?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
WIP FPC-III support
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / gt / intel_rps.c
blobb629eeb1400297c08a90ff73a42fec93601f35f2
1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7 #include <drm/i915_drm.h>
8
9 #include "i915_drv.h"
10 #include "intel_breadcrumbs.h"
11 #include "intel_gt.h"
12 #include "intel_gt_clock_utils.h"
13 #include "intel_gt_irq.h"
14 #include "intel_gt_pm_irq.h"
15 #include "intel_rps.h"
16 #include "intel_sideband.h"
17 #include "../../../platform/x86/intel_ips.h"
18
19 #define BUSY_MAX_EI 20u /* ms */
20
21 /*
22 * Lock protecting IPS related data structures
23 */
24 static DEFINE_SPINLOCK(mchdev_lock);
25
26 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
27 {
28 return container_of(rps, struct intel_gt, rps);
29 }
30
31 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
32 {
33 return rps_to_gt(rps)->i915;
34 }
35
36 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
37 {
38 return rps_to_gt(rps)->uncore;
39 }
40
41 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
42 {
43 return mask & ~rps->pm_intrmsk_mbz;
44 }
45
46 static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
47 {
48 intel_uncore_write_fw(uncore, reg, val);
49 }
50
51 static void rps_timer(struct timer_list *t)
52 {
53 struct intel_rps *rps = from_timer(rps, t, timer);
54 struct intel_engine_cs *engine;
55 ktime_t dt, last, timestamp;
56 enum intel_engine_id id;
57 s64 max_busy[3] = {};
58
59 timestamp = 0;
60 for_each_engine(engine, rps_to_gt(rps), id) {
61 s64 busy;
62 int i;
63
64 dt = intel_engine_get_busy_time(engine, &timestamp);
65 last = engine->stats.rps;
66 engine->stats.rps = dt;
67
68 busy = ktime_to_ns(ktime_sub(dt, last));
69 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
70 if (busy > max_busy[i])
71 swap(busy, max_busy[i]);
72 }
73 }
74 last = rps->pm_timestamp;
75 rps->pm_timestamp = timestamp;
76
77 if (intel_rps_is_active(rps)) {
78 s64 busy;
79 int i;
80
81 dt = ktime_sub(timestamp, last);
82
83 /*
84 * Our goal is to evaluate each engine independently, so we run
85 * at the lowest clocks required to sustain the heaviest
86 * workload. However, a task may be split into sequential
87 * dependent operations across a set of engines, such that
88 * the independent contributions do not account for high load,
89 * but overall the task is GPU bound. For example, consider
90 * video decode on vcs followed by colour post-processing
91 * on vecs, followed by general post-processing on rcs.
92 * Since multi-engines being active does imply a single
93 * continuous workload across all engines, we hedge our
94 * bets by only contributing a factor of the distributed
95 * load into our busyness calculation.
96 */
97 busy = max_busy[0];
98 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
99 if (!max_busy[i])
100 break;
101
102 busy += div_u64(max_busy[i], 1 << i);
103 }
104 GT_TRACE(rps_to_gt(rps),
105 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
106 busy, (int)div64_u64(100 * busy, dt),
107 max_busy[0], max_busy[1], max_busy[2],
108 rps->pm_interval);
109
110 if (100 * busy > rps->power.up_threshold * dt &&
111 rps->cur_freq < rps->max_freq_softlimit) {
112 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
113 rps->pm_interval = 1;
114 schedule_work(&rps->work);
115 } else if (100 * busy < rps->power.down_threshold * dt &&
116 rps->cur_freq > rps->min_freq_softlimit) {
117 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
118 rps->pm_interval = 1;
119 schedule_work(&rps->work);
120 } else {
121 rps->last_adj = 0;
122 }
123
124 mod_timer(&rps->timer,
125 jiffies + msecs_to_jiffies(rps->pm_interval));
126 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
127 }
128 }
129
130 static void rps_start_timer(struct intel_rps *rps)
131 {
132 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
133 rps->pm_interval = 1;
134 mod_timer(&rps->timer, jiffies + 1);
135 }
136
137 static void rps_stop_timer(struct intel_rps *rps)
138 {
139 del_timer_sync(&rps->timer);
140 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
141 cancel_work_sync(&rps->work);
142 }
143
144 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
145 {
146 u32 mask = 0;
147
148 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
149 if (val > rps->min_freq_softlimit)
150 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
151 GEN6_PM_RP_DOWN_THRESHOLD |
152 GEN6_PM_RP_DOWN_TIMEOUT);
153
154 if (val < rps->max_freq_softlimit)
155 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
156
157 mask &= rps->pm_events;
158
159 return rps_pm_sanitize_mask(rps, ~mask);
160 }
161
162 static void rps_reset_ei(struct intel_rps *rps)
163 {
164 memset(&rps->ei, 0, sizeof(rps->ei));
165 }
166
167 static void rps_enable_interrupts(struct intel_rps *rps)
168 {
169 struct intel_gt *gt = rps_to_gt(rps);
170
171 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
172 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
173
174 rps_reset_ei(rps);
175
176 spin_lock_irq(&gt->irq_lock);
177 gen6_gt_pm_enable_irq(gt, rps->pm_events);
178 spin_unlock_irq(&gt->irq_lock);
179
180 intel_uncore_write(gt->uncore,
181 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
182 }
183
184 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
185 {
186 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
187 }
188
189 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
190 {
191 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
192 ;
193 }
194
195 static void rps_reset_interrupts(struct intel_rps *rps)
196 {
197 struct intel_gt *gt = rps_to_gt(rps);
198
199 spin_lock_irq(&gt->irq_lock);
200 if (INTEL_GEN(gt->i915) >= 11)
201 gen11_rps_reset_interrupts(rps);
202 else
203 gen6_rps_reset_interrupts(rps);
204
205 rps->pm_iir = 0;
206 spin_unlock_irq(&gt->irq_lock);
207 }
208
209 static void rps_disable_interrupts(struct intel_rps *rps)
210 {
211 struct intel_gt *gt = rps_to_gt(rps);
212
213 intel_uncore_write(gt->uncore,
214 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
215
216 spin_lock_irq(&gt->irq_lock);
217 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
218 spin_unlock_irq(&gt->irq_lock);
219
220 intel_synchronize_irq(gt->i915);
221
222 /*
223 * Now that we will not be generating any more work, flush any
224 * outstanding tasks. As we are called on the RPS idle path,
225 * we will reset the GPU to minimum frequencies, so the current
226 * state of the worker can be discarded.
227 */
228 cancel_work_sync(&rps->work);
229
230 rps_reset_interrupts(rps);
231 GT_TRACE(gt, "interrupts:off\n");
232 }
233
234 static const struct cparams {
235 u16 i;
236 u16 t;
237 u16 m;
238 u16 c;
239 } cparams[] = {
240 { 1, 1333, 301, 28664 },
241 { 1, 1066, 294, 24460 },
242 { 1, 800, 294, 25192 },
243 { 0, 1333, 276, 27605 },
244 { 0, 1066, 276, 27605 },
245 { 0, 800, 231, 23784 },
246 };
247
248 static void gen5_rps_init(struct intel_rps *rps)
249 {
250 struct drm_i915_private *i915 = rps_to_i915(rps);
251 struct intel_uncore *uncore = rps_to_uncore(rps);
252 u8 fmax, fmin, fstart;
253 u32 rgvmodectl;
254 int c_m, i;
255
256 if (i915->fsb_freq <= 3200)
257 c_m = 0;
258 else if (i915->fsb_freq <= 4800)
259 c_m = 1;
260 else
261 c_m = 2;
262
263 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
264 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
265 rps->ips.m = cparams[i].m;
266 rps->ips.c = cparams[i].c;
267 break;
268 }
269 }
270
271 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
272
273 /* Set up min, max, and cur for interrupt handling */
274 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
275 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
276 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
277 MEMMODE_FSTART_SHIFT;
278 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
279 fmax, fmin, fstart);
280
281 rps->min_freq = fmax;
282 rps->efficient_freq = fstart;
283 rps->max_freq = fmin;
284 }
285
286 static unsigned long
287 __ips_chipset_val(struct intel_ips *ips)
288 {
289 struct intel_uncore *uncore =
290 rps_to_uncore(container_of(ips, struct intel_rps, ips));
291 unsigned long now = jiffies_to_msecs(jiffies), dt;
292 unsigned long result;
293 u64 total, delta;
294
295 lockdep_assert_held(&mchdev_lock);
296
297 /*
298 * Prevent division-by-zero if we are asking too fast.
299 * Also, we don't get interesting results if we are polling
300 * faster than once in 10ms, so just return the saved value
301 * in such cases.
302 */
303 dt = now - ips->last_time1;
304 if (dt <= 10)
305 return ips->chipset_power;
306
307 /* FIXME: handle per-counter overflow */
308 total = intel_uncore_read(uncore, DMIEC);
309 total += intel_uncore_read(uncore, DDREC);
310 total += intel_uncore_read(uncore, CSIEC);
311
312 delta = total - ips->last_count1;
313
314 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
315
316 ips->last_count1 = total;
317 ips->last_time1 = now;
318
319 ips->chipset_power = result;
320
321 return result;
322 }
323
324 static unsigned long ips_mch_val(struct intel_uncore *uncore)
325 {
326 unsigned int m, x, b;
327 u32 tsfs;
328
329 tsfs = intel_uncore_read(uncore, TSFS);
330 x = intel_uncore_read8(uncore, TR1);
331
332 b = tsfs & TSFS_INTR_MASK;
333 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
334
335 return m * x / 127 - b;
336 }
337
338 static int _pxvid_to_vd(u8 pxvid)
339 {
340 if (pxvid == 0)
341 return 0;
342
343 if (pxvid >= 8 && pxvid < 31)
344 pxvid = 31;
345
346 return (pxvid + 2) * 125;
347 }
348
349 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
350 {
351 const int vd = _pxvid_to_vd(pxvid);
352
353 if (INTEL_INFO(i915)->is_mobile)
354 return max(vd - 1125, 0);
355
356 return vd;
357 }
358
359 static void __gen5_ips_update(struct intel_ips *ips)
360 {
361 struct intel_uncore *uncore =
362 rps_to_uncore(container_of(ips, struct intel_rps, ips));
363 u64 now, delta, dt;
364 u32 count;
365
366 lockdep_assert_held(&mchdev_lock);
367
368 now = ktime_get_raw_ns();
369 dt = now - ips->last_time2;
370 do_div(dt, NSEC_PER_MSEC);
371
372 /* Don't divide by 0 */
373 if (dt <= 10)
374 return;
375
376 count = intel_uncore_read(uncore, GFXEC);
377 delta = count - ips->last_count2;
378
379 ips->last_count2 = count;
380 ips->last_time2 = now;
381
382 /* More magic constants... */
383 ips->gfx_power = div_u64(delta * 1181, dt * 10);
384 }
385
386 static void gen5_rps_update(struct intel_rps *rps)
387 {
388 spin_lock_irq(&mchdev_lock);
389 __gen5_ips_update(&rps->ips);
390 spin_unlock_irq(&mchdev_lock);
391 }
392
393 static unsigned int gen5_invert_freq(struct intel_rps *rps,
394 unsigned int val)
395 {
396 /* Invert the frequency bin into an ips delay */
397 val = rps->max_freq - val;
398 val = rps->min_freq + val;
399
400 return val;
401 }
402
403 static bool gen5_rps_set(struct intel_rps *rps, u8 val)
404 {
405 struct intel_uncore *uncore = rps_to_uncore(rps);
406 u16 rgvswctl;
407
408 lockdep_assert_held(&mchdev_lock);
409
410 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
411 if (rgvswctl & MEMCTL_CMD_STS) {
412 DRM_DEBUG("gpu busy, RCS change rejected\n");
413 return false; /* still busy with another command */
414 }
415
416 /* Invert the frequency bin into an ips delay */
417 val = gen5_invert_freq(rps, val);
418
419 rgvswctl =
420 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
421 (val << MEMCTL_FREQ_SHIFT) |
422 MEMCTL_SFCAVM;
423 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
424 intel_uncore_posting_read16(uncore, MEMSWCTL);
425
426 rgvswctl |= MEMCTL_CMD_STS;
427 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
428
429 return true;
430 }
431
432 static unsigned long intel_pxfreq(u32 vidfreq)
433 {
434 int div = (vidfreq & 0x3f0000) >> 16;
435 int post = (vidfreq & 0x3000) >> 12;
436 int pre = (vidfreq & 0x7);
437
438 if (!pre)
439 return 0;
440
441 return div * 133333 / (pre << post);
442 }
443
444 static unsigned int init_emon(struct intel_uncore *uncore)
445 {
446 u8 pxw[16];
447 int i;
448
449 /* Disable to program */
450 intel_uncore_write(uncore, ECR, 0);
451 intel_uncore_posting_read(uncore, ECR);
452
453 /* Program energy weights for various events */
454 intel_uncore_write(uncore, SDEW, 0x15040d00);
455 intel_uncore_write(uncore, CSIEW0, 0x007f0000);
456 intel_uncore_write(uncore, CSIEW1, 0x1e220004);
457 intel_uncore_write(uncore, CSIEW2, 0x04000004);
458
459 for (i = 0; i < 5; i++)
460 intel_uncore_write(uncore, PEW(i), 0);
461 for (i = 0; i < 3; i++)
462 intel_uncore_write(uncore, DEW(i), 0);
463
464 /* Program P-state weights to account for frequency power adjustment */
465 for (i = 0; i < 16; i++) {
466 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
467 unsigned int freq = intel_pxfreq(pxvidfreq);
468 unsigned int vid =
469 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
470 unsigned int val;
471
472 val = vid * vid * freq / 1000 * 255;
473 val /= 127 * 127 * 900;
474
475 pxw[i] = val;
476 }
477 /* Render standby states get 0 weight */
478 pxw[14] = 0;
479 pxw[15] = 0;
480
481 for (i = 0; i < 4; i++) {
482 intel_uncore_write(uncore, PXW(i),
483 pxw[i * 4 + 0] << 24 |
484 pxw[i * 4 + 1] << 16 |
485 pxw[i * 4 + 2] << 8 |
486 pxw[i * 4 + 3] << 0);
487 }
488
489 /* Adjust magic regs to magic values (more experimental results) */
490 intel_uncore_write(uncore, OGW0, 0);
491 intel_uncore_write(uncore, OGW1, 0);
492 intel_uncore_write(uncore, EG0, 0x00007f00);
493 intel_uncore_write(uncore, EG1, 0x0000000e);
494 intel_uncore_write(uncore, EG2, 0x000e0000);
495 intel_uncore_write(uncore, EG3, 0x68000300);
496 intel_uncore_write(uncore, EG4, 0x42000000);
497 intel_uncore_write(uncore, EG5, 0x00140031);
498 intel_uncore_write(uncore, EG6, 0);
499 intel_uncore_write(uncore, EG7, 0);
500
501 for (i = 0; i < 8; i++)
502 intel_uncore_write(uncore, PXWL(i), 0);
503
504 /* Enable PMON + select events */
505 intel_uncore_write(uncore, ECR, 0x80000019);
506
507 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
508 }
509
510 static bool gen5_rps_enable(struct intel_rps *rps)
511 {
512 struct drm_i915_private *i915 = rps_to_i915(rps);
513 struct intel_uncore *uncore = rps_to_uncore(rps);
514 u8 fstart, vstart;
515 u32 rgvmodectl;
516
517 spin_lock_irq(&mchdev_lock);
518
519 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
520
521 /* Enable temp reporting */
522 intel_uncore_write16(uncore, PMMISC,
523 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
524 intel_uncore_write16(uncore, TSC1,
525 intel_uncore_read16(uncore, TSC1) | TSE);
526
527 /* 100ms RC evaluation intervals */
528 intel_uncore_write(uncore, RCUPEI, 100000);
529 intel_uncore_write(uncore, RCDNEI, 100000);
530
531 /* Set max/min thresholds to 90ms and 80ms respectively */
532 intel_uncore_write(uncore, RCBMAXAVG, 90000);
533 intel_uncore_write(uncore, RCBMINAVG, 80000);
534
535 intel_uncore_write(uncore, MEMIHYST, 1);
536
537 /* Set up min, max, and cur for interrupt handling */
538 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
539 MEMMODE_FSTART_SHIFT;
540
541 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
542 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
543
544 intel_uncore_write(uncore,
545 MEMINTREN,
546 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
547
548 intel_uncore_write(uncore, VIDSTART, vstart);
549 intel_uncore_posting_read(uncore, VIDSTART);
550
551 rgvmodectl |= MEMMODE_SWMODE_EN;
552 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
553
554 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
555 MEMCTL_CMD_STS) == 0, 10))
556 drm_err(&uncore->i915->drm,
557 "stuck trying to change perf mode\n");
558 mdelay(1);
559
560 gen5_rps_set(rps, rps->cur_freq);
561
562 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
563 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
564 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
565 rps->ips.last_time1 = jiffies_to_msecs(jiffies);
566
567 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
568 rps->ips.last_time2 = ktime_get_raw_ns();
569
570 spin_lock(&i915->irq_lock);
571 ilk_enable_display_irq(i915, DE_PCU_EVENT);
572 spin_unlock(&i915->irq_lock);
573
574 spin_unlock_irq(&mchdev_lock);
575
576 rps->ips.corr = init_emon(uncore);
577
578 return true;
579 }
580
581 static void gen5_rps_disable(struct intel_rps *rps)
582 {
583 struct drm_i915_private *i915 = rps_to_i915(rps);
584 struct intel_uncore *uncore = rps_to_uncore(rps);
585 u16 rgvswctl;
586
587 spin_lock_irq(&mchdev_lock);
588
589 spin_lock(&i915->irq_lock);
590 ilk_disable_display_irq(i915, DE_PCU_EVENT);
591 spin_unlock(&i915->irq_lock);
592
593 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
594
595 /* Ack interrupts, disable EFC interrupt */
596 intel_uncore_write(uncore, MEMINTREN,
597 intel_uncore_read(uncore, MEMINTREN) &
598 ~MEMINT_EVAL_CHG_EN);
599 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
600
601 /* Go back to the starting frequency */
602 gen5_rps_set(rps, rps->idle_freq);
603 mdelay(1);
604 rgvswctl |= MEMCTL_CMD_STS;
605 intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
606 mdelay(1);
607
608 spin_unlock_irq(&mchdev_lock);
609 }
610
611 static u32 rps_limits(struct intel_rps *rps, u8 val)
612 {
613 u32 limits;
614
615 /*
616 * Only set the down limit when we've reached the lowest level to avoid
617 * getting more interrupts, otherwise leave this clear. This prevents a
618 * race in the hw when coming out of rc6: There's a tiny window where
619 * the hw runs at the minimal clock before selecting the desired
620 * frequency, if the down threshold expires in that window we will not
621 * receive a down interrupt.
622 */
623 if (INTEL_GEN(rps_to_i915(rps)) >= 9) {
624 limits = rps->max_freq_softlimit << 23;
625 if (val <= rps->min_freq_softlimit)
626 limits |= rps->min_freq_softlimit << 14;
627 } else {
628 limits = rps->max_freq_softlimit << 24;
629 if (val <= rps->min_freq_softlimit)
630 limits |= rps->min_freq_softlimit << 16;
631 }
632
633 return limits;
634 }
635
636 static void rps_set_power(struct intel_rps *rps, int new_power)
637 {
638 struct intel_gt *gt = rps_to_gt(rps);
639 struct intel_uncore *uncore = gt->uncore;
640 u32 threshold_up = 0, threshold_down = 0; /* in % */
641 u32 ei_up = 0, ei_down = 0;
642
643 lockdep_assert_held(&rps->power.mutex);
644
645 if (new_power == rps->power.mode)
646 return;
647
648 threshold_up = 95;
649 threshold_down = 85;
650
651 /* Note the units here are not exactly 1us, but 1280ns. */
652 switch (new_power) {
653 case LOW_POWER:
654 ei_up = 16000;
655 ei_down = 32000;
656 break;
657
658 case BETWEEN:
659 ei_up = 13000;
660 ei_down = 32000;
661 break;
662
663 case HIGH_POWER:
664 ei_up = 10000;
665 ei_down = 32000;
666 break;
667 }
668
669 /* When byt can survive without system hang with dynamic
670 * sw freq adjustments, this restriction can be lifted.
671 */
672 if (IS_VALLEYVIEW(gt->i915))
673 goto skip_hw_write;
674
675 GT_TRACE(gt,
676 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
677 new_power, threshold_up, ei_up, threshold_down, ei_down);
678
679 set(uncore, GEN6_RP_UP_EI,
680 intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
681 set(uncore, GEN6_RP_UP_THRESHOLD,
682 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
683
684 set(uncore, GEN6_RP_DOWN_EI,
685 intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
686 set(uncore, GEN6_RP_DOWN_THRESHOLD,
687 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
688
689 set(uncore, GEN6_RP_CONTROL,
690 (INTEL_GEN(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
691 GEN6_RP_MEDIA_HW_NORMAL_MODE |
692 GEN6_RP_MEDIA_IS_GFX |
693 GEN6_RP_ENABLE |
694 GEN6_RP_UP_BUSY_AVG |
695 GEN6_RP_DOWN_IDLE_AVG);
696
697 skip_hw_write:
698 rps->power.mode = new_power;
699 rps->power.up_threshold = threshold_up;
700 rps->power.down_threshold = threshold_down;
701 }
702
703 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
704 {
705 int new_power;
706
707 new_power = rps->power.mode;
708 switch (rps->power.mode) {
709 case LOW_POWER:
710 if (val > rps->efficient_freq + 1 &&
711 val > rps->cur_freq)
712 new_power = BETWEEN;
713 break;
714
715 case BETWEEN:
716 if (val <= rps->efficient_freq &&
717 val < rps->cur_freq)
718 new_power = LOW_POWER;
719 else if (val >= rps->rp0_freq &&
720 val > rps->cur_freq)
721 new_power = HIGH_POWER;
722 break;
723
724 case HIGH_POWER:
725 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
726 val < rps->cur_freq)
727 new_power = BETWEEN;
728 break;
729 }
730 /* Max/min bins are special */
731 if (val <= rps->min_freq_softlimit)
732 new_power = LOW_POWER;
733 if (val >= rps->max_freq_softlimit)
734 new_power = HIGH_POWER;
735
736 mutex_lock(&rps->power.mutex);
737 if (rps->power.interactive)
738 new_power = HIGH_POWER;
739 rps_set_power(rps, new_power);
740 mutex_unlock(&rps->power.mutex);
741 }
742
743 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
744 {
745 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive));
746
747 mutex_lock(&rps->power.mutex);
748 if (interactive) {
749 if (!rps->power.interactive++ && intel_rps_is_active(rps))
750 rps_set_power(rps, HIGH_POWER);
751 } else {
752 GEM_BUG_ON(!rps->power.interactive);
753 rps->power.interactive--;
754 }
755 mutex_unlock(&rps->power.mutex);
756 }
757
758 static int gen6_rps_set(struct intel_rps *rps, u8 val)
759 {
760 struct intel_uncore *uncore = rps_to_uncore(rps);
761 struct drm_i915_private *i915 = rps_to_i915(rps);
762 u32 swreq;
763
764 if (INTEL_GEN(i915) >= 9)
765 swreq = GEN9_FREQUENCY(val);
766 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
767 swreq = HSW_FREQUENCY(val);
768 else
769 swreq = (GEN6_FREQUENCY(val) |
770 GEN6_OFFSET(0) |
771 GEN6_AGGRESSIVE_TURBO);
772 set(uncore, GEN6_RPNSWREQ, swreq);
773
774 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
775 val, intel_gpu_freq(rps, val), swreq);
776
777 return 0;
778 }
779
780 static int vlv_rps_set(struct intel_rps *rps, u8 val)
781 {
782 struct drm_i915_private *i915 = rps_to_i915(rps);
783 int err;
784
785 vlv_punit_get(i915);
786 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
787 vlv_punit_put(i915);
788
789 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
790 val, intel_gpu_freq(rps, val));
791
792 return err;
793 }
794
795 static int rps_set(struct intel_rps *rps, u8 val, bool update)
796 {
797 struct drm_i915_private *i915 = rps_to_i915(rps);
798 int err;
799
800 if (INTEL_GEN(i915) < 6)
801 return 0;
802
803 if (val == rps->last_freq)
804 return 0;
805
806 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
807 err = vlv_rps_set(rps, val);
808 else
809 err = gen6_rps_set(rps, val);
810 if (err)
811 return err;
812
813 if (update)
814 gen6_rps_set_thresholds(rps, val);
815 rps->last_freq = val;
816
817 return 0;
818 }
819
820 void intel_rps_unpark(struct intel_rps *rps)
821 {
822 if (!intel_rps_is_enabled(rps))
823 return;
824
825 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
826
827 /*
828 * Use the user's desired frequency as a guide, but for better
829 * performance, jump directly to RPe as our starting frequency.
830 */
831 mutex_lock(&rps->lock);
832
833 intel_rps_set_active(rps);
834 intel_rps_set(rps,
835 clamp(rps->cur_freq,
836 rps->min_freq_softlimit,
837 rps->max_freq_softlimit));
838
839 mutex_unlock(&rps->lock);
840
841 rps->pm_iir = 0;
842 if (intel_rps_has_interrupts(rps))
843 rps_enable_interrupts(rps);
844 if (intel_rps_uses_timer(rps))
845 rps_start_timer(rps);
846
847 if (IS_GEN(rps_to_i915(rps), 5))
848 gen5_rps_update(rps);
849 }
850
851 void intel_rps_park(struct intel_rps *rps)
852 {
853 int adj;
854
855 if (!intel_rps_clear_active(rps))
856 return;
857
858 if (intel_rps_uses_timer(rps))
859 rps_stop_timer(rps);
860 if (intel_rps_has_interrupts(rps))
861 rps_disable_interrupts(rps);
862
863 if (rps->last_freq <= rps->idle_freq)
864 return;
865
866 /*
867 * The punit delays the write of the frequency and voltage until it
868 * determines the GPU is awake. During normal usage we don't want to
869 * waste power changing the frequency if the GPU is sleeping (rc6).
870 * However, the GPU and driver is now idle and we do not want to delay
871 * switching to minimum voltage (reducing power whilst idle) as we do
872 * not expect to be woken in the near future and so must flush the
873 * change by waking the device.
874 *
875 * We choose to take the media powerwell (either would do to trick the
876 * punit into committing the voltage change) as that takes a lot less
877 * power than the render powerwell.
878 */
879 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
880 rps_set(rps, rps->idle_freq, false);
881 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
882
883 /*
884 * Since we will try and restart from the previously requested
885 * frequency on unparking, treat this idle point as a downclock
886 * interrupt and reduce the frequency for resume. If we park/unpark
887 * more frequently than the rps worker can run, we will not respond
888 * to any EI and never see a change in frequency.
889 *
890 * (Note we accommodate Cherryview's limitation of only using an
891 * even bin by applying it to all.)
892 */
893 adj = rps->last_adj;
894 if (adj < 0)
895 adj *= 2;
896 else /* CHV needs even encode values */
897 adj = -2;
898 rps->last_adj = adj;
899 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
900 if (rps->cur_freq < rps->efficient_freq) {
901 rps->cur_freq = rps->efficient_freq;
902 rps->last_adj = 0;
903 }
904
905 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
906 }
907
908 void intel_rps_boost(struct i915_request *rq)
909 {
910 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
911 unsigned long flags;
912
913 if (i915_request_signaled(rq) || !intel_rps_is_active(rps))
914 return;
915
916 /* Serializes with i915_request_retire() */
917 spin_lock_irqsave(&rq->lock, flags);
918 if (!i915_request_has_waitboost(rq) &&
919 !dma_fence_is_signaled_locked(&rq->fence)) {
920 set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags);
921
922 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
923 rq->fence.context, rq->fence.seqno);
924
925 if (!atomic_fetch_inc(&rps->num_waiters) &&
926 READ_ONCE(rps->cur_freq) < rps->boost_freq)
927 schedule_work(&rps->work);
928
929 atomic_inc(&rps->boosts);
930 }
931 spin_unlock_irqrestore(&rq->lock, flags);
932 }
933
934 int intel_rps_set(struct intel_rps *rps, u8 val)
935 {
936 int err;
937
938 lockdep_assert_held(&rps->lock);
939 GEM_BUG_ON(val > rps->max_freq);
940 GEM_BUG_ON(val < rps->min_freq);
941
942 if (intel_rps_is_active(rps)) {
943 err = rps_set(rps, val, true);
944 if (err)
945 return err;
946
947 /*
948 * Make sure we continue to get interrupts
949 * until we hit the minimum or maximum frequencies.
950 */
951 if (intel_rps_has_interrupts(rps)) {
952 struct intel_uncore *uncore = rps_to_uncore(rps);
953
954 set(uncore,
955 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
956
957 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
958 }
959 }
960
961 rps->cur_freq = val;
962 return 0;
963 }
964
965 static void gen6_rps_init(struct intel_rps *rps)
966 {
967 struct drm_i915_private *i915 = rps_to_i915(rps);
968 struct intel_uncore *uncore = rps_to_uncore(rps);
969
970 /* All of these values are in units of 50MHz */
971
972 /* static values from HW: RP0 > RP1 > RPn (min_freq) */
973 if (IS_GEN9_LP(i915)) {
974 u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP);
975
976 rps->rp0_freq = (rp_state_cap >> 16) & 0xff;
977 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
978 rps->min_freq = (rp_state_cap >> 0) & 0xff;
979 } else {
980 u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
981
982 rps->rp0_freq = (rp_state_cap >> 0) & 0xff;
983 rps->rp1_freq = (rp_state_cap >> 8) & 0xff;
984 rps->min_freq = (rp_state_cap >> 16) & 0xff;
985 }
986
987 /* hw_max = RP0 until we check for overclocking */
988 rps->max_freq = rps->rp0_freq;
989
990 rps->efficient_freq = rps->rp1_freq;
991 if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
992 IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
993 u32 ddcc_status = 0;
994
995 if (sandybridge_pcode_read(i915,
996 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
997 &ddcc_status, NULL) == 0)
998 rps->efficient_freq =
999 clamp_t(u8,
1000 (ddcc_status >> 8) & 0xff,
1001 rps->min_freq,
1002 rps->max_freq);
1003 }
1004
1005 if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) {
1006 /* Store the frequency values in 16.66 MHZ units, which is
1007 * the natural hardware unit for SKL
1008 */
1009 rps->rp0_freq *= GEN9_FREQ_SCALER;
1010 rps->rp1_freq *= GEN9_FREQ_SCALER;
1011 rps->min_freq *= GEN9_FREQ_SCALER;
1012 rps->max_freq *= GEN9_FREQ_SCALER;
1013 rps->efficient_freq *= GEN9_FREQ_SCALER;
1014 }
1015 }
1016
1017 static bool rps_reset(struct intel_rps *rps)
1018 {
1019 struct drm_i915_private *i915 = rps_to_i915(rps);
1020
1021 /* force a reset */
1022 rps->power.mode = -1;
1023 rps->last_freq = -1;
1024
1025 if (rps_set(rps, rps->min_freq, true)) {
1026 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1027 return false;
1028 }
1029
1030 rps->cur_freq = rps->min_freq;
1031 return true;
1032 }
1033
1034 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1035 static bool gen9_rps_enable(struct intel_rps *rps)
1036 {
1037 struct intel_gt *gt = rps_to_gt(rps);
1038 struct intel_uncore *uncore = gt->uncore;
1039
1040 /* Program defaults and thresholds for RPS */
1041 if (IS_GEN(gt->i915, 9))
1042 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1043 GEN9_FREQUENCY(rps->rp1_freq));
1044
1045 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1046
1047 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1048
1049 return rps_reset(rps);
1050 }
1051
1052 static bool gen8_rps_enable(struct intel_rps *rps)
1053 {
1054 struct intel_uncore *uncore = rps_to_uncore(rps);
1055
1056 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1057 HSW_FREQUENCY(rps->rp1_freq));
1058
1059 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1060
1061 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1062
1063 return rps_reset(rps);
1064 }
1065
1066 static bool gen6_rps_enable(struct intel_rps *rps)
1067 {
1068 struct intel_uncore *uncore = rps_to_uncore(rps);
1069
1070 /* Power down if completely idle for over 50ms */
1071 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1072 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1073
1074 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1075 GEN6_PM_RP_DOWN_THRESHOLD |
1076 GEN6_PM_RP_DOWN_TIMEOUT);
1077
1078 return rps_reset(rps);
1079 }
1080
1081 static int chv_rps_max_freq(struct intel_rps *rps)
1082 {
1083 struct drm_i915_private *i915 = rps_to_i915(rps);
1084 struct intel_gt *gt = rps_to_gt(rps);
1085 u32 val;
1086
1087 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1088
1089 switch (gt->info.sseu.eu_total) {
1090 case 8:
1091 /* (2 * 4) config */
1092 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1093 break;
1094 case 12:
1095 /* (2 * 6) config */
1096 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1097 break;
1098 case 16:
1099 /* (2 * 8) config */
1100 default:
1101 /* Setting (2 * 8) Min RP0 for any other combination */
1102 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1103 break;
1104 }
1105
1106 return val & FB_GFX_FREQ_FUSE_MASK;
1107 }
1108
1109 static int chv_rps_rpe_freq(struct intel_rps *rps)
1110 {
1111 struct drm_i915_private *i915 = rps_to_i915(rps);
1112 u32 val;
1113
1114 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1115 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1116
1117 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1118 }
1119
1120 static int chv_rps_guar_freq(struct intel_rps *rps)
1121 {
1122 struct drm_i915_private *i915 = rps_to_i915(rps);
1123 u32 val;
1124
1125 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1126
1127 return val & FB_GFX_FREQ_FUSE_MASK;
1128 }
1129
1130 static u32 chv_rps_min_freq(struct intel_rps *rps)
1131 {
1132 struct drm_i915_private *i915 = rps_to_i915(rps);
1133 u32 val;
1134
1135 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1136 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1137
1138 return val & FB_GFX_FREQ_FUSE_MASK;
1139 }
1140
1141 static bool chv_rps_enable(struct intel_rps *rps)
1142 {
1143 struct intel_uncore *uncore = rps_to_uncore(rps);
1144 struct drm_i915_private *i915 = rps_to_i915(rps);
1145 u32 val;
1146
1147 /* 1: Program defaults and thresholds for RPS*/
1148 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1149 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1150 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1151 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1152 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1153
1154 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1155
1156 /* 2: Enable RPS */
1157 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1158 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1159 GEN6_RP_MEDIA_IS_GFX |
1160 GEN6_RP_ENABLE |
1161 GEN6_RP_UP_BUSY_AVG |
1162 GEN6_RP_DOWN_IDLE_AVG);
1163
1164 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1165 GEN6_PM_RP_DOWN_THRESHOLD |
1166 GEN6_PM_RP_DOWN_TIMEOUT);
1167
1168 /* Setting Fixed Bias */
1169 vlv_punit_get(i915);
1170
1171 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1172 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1173
1174 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1175
1176 vlv_punit_put(i915);
1177
1178 /* RPS code assumes GPLL is used */
1179 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1180 "GPLL not enabled\n");
1181
1182 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1183 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1184
1185 return rps_reset(rps);
1186 }
1187
1188 static int vlv_rps_guar_freq(struct intel_rps *rps)
1189 {
1190 struct drm_i915_private *i915 = rps_to_i915(rps);
1191 u32 val, rp1;
1192
1193 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1194
1195 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1196 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1197
1198 return rp1;
1199 }
1200
1201 static int vlv_rps_max_freq(struct intel_rps *rps)
1202 {
1203 struct drm_i915_private *i915 = rps_to_i915(rps);
1204 u32 val, rp0;
1205
1206 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1207
1208 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1209 /* Clamp to max */
1210 rp0 = min_t(u32, rp0, 0xea);
1211
1212 return rp0;
1213 }
1214
1215 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1216 {
1217 struct drm_i915_private *i915 = rps_to_i915(rps);
1218 u32 val, rpe;
1219
1220 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1221 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1222 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1223 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1224
1225 return rpe;
1226 }
1227
1228 static int vlv_rps_min_freq(struct intel_rps *rps)
1229 {
1230 struct drm_i915_private *i915 = rps_to_i915(rps);
1231 u32 val;
1232
1233 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1234 /*
1235 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1236 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1237 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1238 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1239 * to make sure it matches what Punit accepts.
1240 */
1241 return max_t(u32, val, 0xc0);
1242 }
1243
1244 static bool vlv_rps_enable(struct intel_rps *rps)
1245 {
1246 struct intel_uncore *uncore = rps_to_uncore(rps);
1247 struct drm_i915_private *i915 = rps_to_i915(rps);
1248 u32 val;
1249
1250 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1251 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1252 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1253 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1254 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1255
1256 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1257
1258 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1259 GEN6_RP_MEDIA_TURBO |
1260 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1261 GEN6_RP_MEDIA_IS_GFX |
1262 GEN6_RP_ENABLE |
1263 GEN6_RP_UP_BUSY_AVG |
1264 GEN6_RP_DOWN_IDLE_CONT);
1265
1266 /* WaGsvRC0ResidencyMethod:vlv */
1267 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1268
1269 vlv_punit_get(i915);
1270
1271 /* Setting Fixed Bias */
1272 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1273 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1274
1275 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1276
1277 vlv_punit_put(i915);
1278
1279 /* RPS code assumes GPLL is used */
1280 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1281 "GPLL not enabled\n");
1282
1283 drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE));
1284 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1285
1286 return rps_reset(rps);
1287 }
1288
1289 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1290 {
1291 struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1292 struct intel_uncore *uncore = rps_to_uncore(rps);
1293 unsigned int t, state1, state2;
1294 u32 pxvid, ext_v;
1295 u64 corr, corr2;
1296
1297 lockdep_assert_held(&mchdev_lock);
1298
1299 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1300 pxvid = (pxvid >> 24) & 0x7f;
1301 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1302
1303 state1 = ext_v;
1304
1305 /* Revel in the empirically derived constants */
1306
1307 /* Correction factor in 1/100000 units */
1308 t = ips_mch_val(uncore);
1309 if (t > 80)
1310 corr = t * 2349 + 135940;
1311 else if (t >= 50)
1312 corr = t * 964 + 29317;
1313 else /* < 50 */
1314 corr = t * 301 + 1004;
1315
1316 corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1317 corr2 = div_u64(corr, 100000) * ips->corr;
1318
1319 state2 = div_u64(corr2 * state1, 10000);
1320 state2 /= 100; /* convert to mW */
1321
1322 __gen5_ips_update(ips);
1323
1324 return ips->gfx_power + state2;
1325 }
1326
1327 static bool has_busy_stats(struct intel_rps *rps)
1328 {
1329 struct intel_engine_cs *engine;
1330 enum intel_engine_id id;
1331
1332 for_each_engine(engine, rps_to_gt(rps), id) {
1333 if (!intel_engine_supports_stats(engine))
1334 return false;
1335 }
1336
1337 return true;
1338 }
1339
1340 void intel_rps_enable(struct intel_rps *rps)
1341 {
1342 struct drm_i915_private *i915 = rps_to_i915(rps);
1343 struct intel_uncore *uncore = rps_to_uncore(rps);
1344 bool enabled = false;
1345
1346 if (!HAS_RPS(i915))
1347 return;
1348
1349 intel_gt_check_clock_frequency(rps_to_gt(rps));
1350
1351 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1352 if (rps->max_freq <= rps->min_freq)
1353 /* leave disabled, no room for dynamic reclocking */;
1354 else if (IS_CHERRYVIEW(i915))
1355 enabled = chv_rps_enable(rps);
1356 else if (IS_VALLEYVIEW(i915))
1357 enabled = vlv_rps_enable(rps);
1358 else if (INTEL_GEN(i915) >= 9)
1359 enabled = gen9_rps_enable(rps);
1360 else if (INTEL_GEN(i915) >= 8)
1361 enabled = gen8_rps_enable(rps);
1362 else if (INTEL_GEN(i915) >= 6)
1363 enabled = gen6_rps_enable(rps);
1364 else if (IS_IRONLAKE_M(i915))
1365 enabled = gen5_rps_enable(rps);
1366 else
1367 MISSING_CASE(INTEL_GEN(i915));
1368 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1369 if (!enabled)
1370 return;
1371
1372 GT_TRACE(rps_to_gt(rps),
1373 "min:%x, max:%x, freq:[%d, %d]\n",
1374 rps->min_freq, rps->max_freq,
1375 intel_gpu_freq(rps, rps->min_freq),
1376 intel_gpu_freq(rps, rps->max_freq));
1377
1378 GEM_BUG_ON(rps->max_freq < rps->min_freq);
1379 GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1380
1381 GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1382 GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1383
1384 if (has_busy_stats(rps))
1385 intel_rps_set_timer(rps);
1386 else if (INTEL_GEN(i915) >= 6)
1387 intel_rps_set_interrupts(rps);
1388 else
1389 /* Ironlake currently uses intel_ips.ko */ {}
1390
1391 intel_rps_set_enabled(rps);
1392 }
1393
1394 static void gen6_rps_disable(struct intel_rps *rps)
1395 {
1396 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1397 }
1398
1399 void intel_rps_disable(struct intel_rps *rps)
1400 {
1401 struct drm_i915_private *i915 = rps_to_i915(rps);
1402
1403 intel_rps_clear_enabled(rps);
1404 intel_rps_clear_interrupts(rps);
1405 intel_rps_clear_timer(rps);
1406
1407 if (INTEL_GEN(i915) >= 6)
1408 gen6_rps_disable(rps);
1409 else if (IS_IRONLAKE_M(i915))
1410 gen5_rps_disable(rps);
1411 }
1412
1413 static int byt_gpu_freq(struct intel_rps *rps, int val)
1414 {
1415 /*
1416 * N = val - 0xb7
1417 * Slow = Fast = GPLL ref * N
1418 */
1419 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1420 }
1421
1422 static int byt_freq_opcode(struct intel_rps *rps, int val)
1423 {
1424 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1425 }
1426
1427 static int chv_gpu_freq(struct intel_rps *rps, int val)
1428 {
1429 /*
1430 * N = val / 2
1431 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1432 */
1433 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1434 }
1435
1436 static int chv_freq_opcode(struct intel_rps *rps, int val)
1437 {
1438 /* CHV needs even values */
1439 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1440 }
1441
1442 int intel_gpu_freq(struct intel_rps *rps, int val)
1443 {
1444 struct drm_i915_private *i915 = rps_to_i915(rps);
1445
1446 if (INTEL_GEN(i915) >= 9)
1447 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1448 GEN9_FREQ_SCALER);
1449 else if (IS_CHERRYVIEW(i915))
1450 return chv_gpu_freq(rps, val);
1451 else if (IS_VALLEYVIEW(i915))
1452 return byt_gpu_freq(rps, val);
1453 else if (INTEL_GEN(i915) >= 6)
1454 return val * GT_FREQUENCY_MULTIPLIER;
1455 else
1456 return val;
1457 }
1458
1459 int intel_freq_opcode(struct intel_rps *rps, int val)
1460 {
1461 struct drm_i915_private *i915 = rps_to_i915(rps);
1462
1463 if (INTEL_GEN(i915) >= 9)
1464 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1465 GT_FREQUENCY_MULTIPLIER);
1466 else if (IS_CHERRYVIEW(i915))
1467 return chv_freq_opcode(rps, val);
1468 else if (IS_VALLEYVIEW(i915))
1469 return byt_freq_opcode(rps, val);
1470 else if (INTEL_GEN(i915) >= 6)
1471 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1472 else
1473 return val;
1474 }
1475
1476 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1477 {
1478 struct drm_i915_private *i915 = rps_to_i915(rps);
1479
1480 rps->gpll_ref_freq =
1481 vlv_get_cck_clock(i915, "GPLL ref",
1482 CCK_GPLL_CLOCK_CONTROL,
1483 i915->czclk_freq);
1484
1485 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1486 rps->gpll_ref_freq);
1487 }
1488
1489 static void vlv_rps_init(struct intel_rps *rps)
1490 {
1491 struct drm_i915_private *i915 = rps_to_i915(rps);
1492 u32 val;
1493
1494 vlv_iosf_sb_get(i915,
1495 BIT(VLV_IOSF_SB_PUNIT) |
1496 BIT(VLV_IOSF_SB_NC) |
1497 BIT(VLV_IOSF_SB_CCK));
1498
1499 vlv_init_gpll_ref_freq(rps);
1500
1501 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1502 switch ((val >> 6) & 3) {
1503 case 0:
1504 case 1:
1505 i915->mem_freq = 800;
1506 break;
1507 case 2:
1508 i915->mem_freq = 1066;
1509 break;
1510 case 3:
1511 i915->mem_freq = 1333;
1512 break;
1513 }
1514 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1515
1516 rps->max_freq = vlv_rps_max_freq(rps);
1517 rps->rp0_freq = rps->max_freq;
1518 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1519 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1520
1521 rps->efficient_freq = vlv_rps_rpe_freq(rps);
1522 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1523 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1524
1525 rps->rp1_freq = vlv_rps_guar_freq(rps);
1526 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1527 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1528
1529 rps->min_freq = vlv_rps_min_freq(rps);
1530 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1531 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1532
1533 vlv_iosf_sb_put(i915,
1534 BIT(VLV_IOSF_SB_PUNIT) |
1535 BIT(VLV_IOSF_SB_NC) |
1536 BIT(VLV_IOSF_SB_CCK));
1537 }
1538
1539 static void chv_rps_init(struct intel_rps *rps)
1540 {
1541 struct drm_i915_private *i915 = rps_to_i915(rps);
1542 u32 val;
1543
1544 vlv_iosf_sb_get(i915,
1545 BIT(VLV_IOSF_SB_PUNIT) |
1546 BIT(VLV_IOSF_SB_NC) |
1547 BIT(VLV_IOSF_SB_CCK));
1548
1549 vlv_init_gpll_ref_freq(rps);
1550
1551 val = vlv_cck_read(i915, CCK_FUSE_REG);
1552
1553 switch ((val >> 2) & 0x7) {
1554 case 3:
1555 i915->mem_freq = 2000;
1556 break;
1557 default:
1558 i915->mem_freq = 1600;
1559 break;
1560 }
1561 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
1562
1563 rps->max_freq = chv_rps_max_freq(rps);
1564 rps->rp0_freq = rps->max_freq;
1565 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1566 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1567
1568 rps->efficient_freq = chv_rps_rpe_freq(rps);
1569 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1570 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1571
1572 rps->rp1_freq = chv_rps_guar_freq(rps);
1573 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1574 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1575
1576 rps->min_freq = chv_rps_min_freq(rps);
1577 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1578 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1579
1580 vlv_iosf_sb_put(i915,
1581 BIT(VLV_IOSF_SB_PUNIT) |
1582 BIT(VLV_IOSF_SB_NC) |
1583 BIT(VLV_IOSF_SB_CCK));
1584
1585 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1586 rps->rp1_freq | rps->min_freq) & 1,
1587 "Odd GPU freq values\n");
1588 }
1589
1590 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1591 {
1592 ei->ktime = ktime_get_raw();
1593 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1594 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1595 }
1596
1597 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1598 {
1599 struct intel_uncore *uncore = rps_to_uncore(rps);
1600 const struct intel_rps_ei *prev = &rps->ei;
1601 struct intel_rps_ei now;
1602 u32 events = 0;
1603
1604 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1605 return 0;
1606
1607 vlv_c0_read(uncore, &now);
1608
1609 if (prev->ktime) {
1610 u64 time, c0;
1611 u32 render, media;
1612
1613 time = ktime_us_delta(now.ktime, prev->ktime);
1614
1615 time *= rps_to_i915(rps)->czclk_freq;
1616
1617 /* Workload can be split between render + media,
1618 * e.g. SwapBuffers being blitted in X after being rendered in
1619 * mesa. To account for this we need to combine both engines
1620 * into our activity counter.
1621 */
1622 render = now.render_c0 - prev->render_c0;
1623 media = now.media_c0 - prev->media_c0;
1624 c0 = max(render, media);
1625 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1626
1627 if (c0 > time * rps->power.up_threshold)
1628 events = GEN6_PM_RP_UP_THRESHOLD;
1629 else if (c0 < time * rps->power.down_threshold)
1630 events = GEN6_PM_RP_DOWN_THRESHOLD;
1631 }
1632
1633 rps->ei = now;
1634 return events;
1635 }
1636
1637 static void rps_work(struct work_struct *work)
1638 {
1639 struct intel_rps *rps = container_of(work, typeof(*rps), work);
1640 struct intel_gt *gt = rps_to_gt(rps);
1641 struct drm_i915_private *i915 = rps_to_i915(rps);
1642 bool client_boost = false;
1643 int new_freq, adj, min, max;
1644 u32 pm_iir = 0;
1645
1646 spin_lock_irq(&gt->irq_lock);
1647 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1648 client_boost = atomic_read(&rps->num_waiters);
1649 spin_unlock_irq(&gt->irq_lock);
1650
1651 /* Make sure we didn't queue anything we're not going to process. */
1652 if (!pm_iir && !client_boost)
1653 goto out;
1654
1655 mutex_lock(&rps->lock);
1656 if (!intel_rps_is_active(rps)) {
1657 mutex_unlock(&rps->lock);
1658 return;
1659 }
1660
1661 pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1662
1663 adj = rps->last_adj;
1664 new_freq = rps->cur_freq;
1665 min = rps->min_freq_softlimit;
1666 max = rps->max_freq_softlimit;
1667 if (client_boost)
1668 max = rps->max_freq;
1669
1670 GT_TRACE(gt,
1671 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1672 pm_iir, yesno(client_boost),
1673 adj, new_freq, min, max);
1674
1675 if (client_boost && new_freq < rps->boost_freq) {
1676 new_freq = rps->boost_freq;
1677 adj = 0;
1678 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1679 if (adj > 0)
1680 adj *= 2;
1681 else /* CHV needs even encode values */
1682 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1683
1684 if (new_freq >= rps->max_freq_softlimit)
1685 adj = 0;
1686 } else if (client_boost) {
1687 adj = 0;
1688 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1689 if (rps->cur_freq > rps->efficient_freq)
1690 new_freq = rps->efficient_freq;
1691 else if (rps->cur_freq > rps->min_freq_softlimit)
1692 new_freq = rps->min_freq_softlimit;
1693 adj = 0;
1694 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1695 if (adj < 0)
1696 adj *= 2;
1697 else /* CHV needs even encode values */
1698 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1699
1700 if (new_freq <= rps->min_freq_softlimit)
1701 adj = 0;
1702 } else { /* unknown event */
1703 adj = 0;
1704 }
1705
1706 /*
1707 * sysfs frequency limits may have snuck in while
1708 * servicing the interrupt
1709 */
1710 new_freq += adj;
1711 new_freq = clamp_t(int, new_freq, min, max);
1712
1713 if (intel_rps_set(rps, new_freq)) {
1714 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1715 adj = 0;
1716 }
1717 rps->last_adj = adj;
1718
1719 mutex_unlock(&rps->lock);
1720
1721 out:
1722 spin_lock_irq(&gt->irq_lock);
1723 gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1724 spin_unlock_irq(&gt->irq_lock);
1725 }
1726
1727 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1728 {
1729 struct intel_gt *gt = rps_to_gt(rps);
1730 const u32 events = rps->pm_events & pm_iir;
1731
1732 lockdep_assert_held(&gt->irq_lock);
1733
1734 if (unlikely(!events))
1735 return;
1736
1737 GT_TRACE(gt, "irq events:%x\n", events);
1738
1739 gen6_gt_pm_mask_irq(gt, events);
1740
1741 rps->pm_iir |= events;
1742 schedule_work(&rps->work);
1743 }
1744
1745 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1746 {
1747 struct intel_gt *gt = rps_to_gt(rps);
1748 u32 events;
1749
1750 events = pm_iir & rps->pm_events;
1751 if (events) {
1752 spin_lock(&gt->irq_lock);
1753
1754 GT_TRACE(gt, "irq events:%x\n", events);
1755
1756 gen6_gt_pm_mask_irq(gt, events);
1757 rps->pm_iir |= events;
1758
1759 schedule_work(&rps->work);
1760 spin_unlock(&gt->irq_lock);
1761 }
1762
1763 if (INTEL_GEN(gt->i915) >= 8)
1764 return;
1765
1766 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1767 intel_engine_signal_breadcrumbs(gt->engine[VECS0]);
1768
1769 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1770 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
1771 }
1772
1773 void gen5_rps_irq_handler(struct intel_rps *rps)
1774 {
1775 struct intel_uncore *uncore = rps_to_uncore(rps);
1776 u32 busy_up, busy_down, max_avg, min_avg;
1777 u8 new_freq;
1778
1779 spin_lock(&mchdev_lock);
1780
1781 intel_uncore_write16(uncore,
1782 MEMINTRSTS,
1783 intel_uncore_read(uncore, MEMINTRSTS));
1784
1785 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1786 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1787 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1788 max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1789 min_avg = intel_uncore_read(uncore, RCBMINAVG);
1790
1791 /* Handle RCS change request from hw */
1792 new_freq = rps->cur_freq;
1793 if (busy_up > max_avg)
1794 new_freq++;
1795 else if (busy_down < min_avg)
1796 new_freq--;
1797 new_freq = clamp(new_freq,
1798 rps->min_freq_softlimit,
1799 rps->max_freq_softlimit);
1800
1801 if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq))
1802 rps->cur_freq = new_freq;
1803
1804 spin_unlock(&mchdev_lock);
1805 }
1806
1807 void intel_rps_init_early(struct intel_rps *rps)
1808 {
1809 mutex_init(&rps->lock);
1810 mutex_init(&rps->power.mutex);
1811
1812 INIT_WORK(&rps->work, rps_work);
1813 timer_setup(&rps->timer, rps_timer, 0);
1814
1815 atomic_set(&rps->num_waiters, 0);
1816 }
1817
1818 void intel_rps_init(struct intel_rps *rps)
1819 {
1820 struct drm_i915_private *i915 = rps_to_i915(rps);
1821
1822 if (IS_CHERRYVIEW(i915))
1823 chv_rps_init(rps);
1824 else if (IS_VALLEYVIEW(i915))
1825 vlv_rps_init(rps);
1826 else if (INTEL_GEN(i915) >= 6)
1827 gen6_rps_init(rps);
1828 else if (IS_IRONLAKE_M(i915))
1829 gen5_rps_init(rps);
1830
1831 /* Derive initial user preferences/limits from the hardware limits */
1832 rps->max_freq_softlimit = rps->max_freq;
1833 rps->min_freq_softlimit = rps->min_freq;
1834
1835 /* After setting max-softlimit, find the overclock max freq */
1836 if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
1837 u32 params = 0;
1838
1839 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS,
1840 &params, NULL);
1841 if (params & BIT(31)) { /* OC supported */
1842 drm_dbg(&i915->drm,
1843 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
1844 (rps->max_freq & 0xff) * 50,
1845 (params & 0xff) * 50);
1846 rps->max_freq = params & 0xff;
1847 }
1848 }
1849
1850 /* Finally allow us to boost to max by default */
1851 rps->boost_freq = rps->max_freq;
1852 rps->idle_freq = rps->min_freq;
1853
1854 /* Start in the middle, from here we will autotune based on workload */
1855 rps->cur_freq = rps->efficient_freq;
1856
1857 rps->pm_intrmsk_mbz = 0;
1858
1859 /*
1860 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
1861 * if GEN6_PM_UP_EI_EXPIRED is masked.
1862 *
1863 * TODO: verify if this can be reproduced on VLV,CHV.
1864 */
1865 if (INTEL_GEN(i915) <= 7)
1866 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
1867
1868 if (INTEL_GEN(i915) >= 8 && INTEL_GEN(i915) < 11)
1869 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
1870 }
1871
1872 void intel_rps_sanitize(struct intel_rps *rps)
1873 {
1874 if (INTEL_GEN(rps_to_i915(rps)) >= 6)
1875 rps_disable_interrupts(rps);
1876 }
1877
1878 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
1879 {
1880 struct drm_i915_private *i915 = rps_to_i915(rps);
1881 u32 cagf;
1882
1883 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
1884 cagf = (rpstat >> 8) & 0xff;
1885 else if (INTEL_GEN(i915) >= 9)
1886 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
1887 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
1888 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
1889 else if (INTEL_GEN(i915) >= 6)
1890 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
1891 else
1892 cagf = gen5_invert_freq(rps, (rpstat & MEMSTAT_PSTATE_MASK) >>
1893 MEMSTAT_PSTATE_SHIFT);
1894
1895 return cagf;
1896 }
1897
1898 static u32 read_cagf(struct intel_rps *rps)
1899 {
1900 struct drm_i915_private *i915 = rps_to_i915(rps);
1901 struct intel_uncore *uncore = rps_to_uncore(rps);
1902 u32 freq;
1903
1904 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
1905 vlv_punit_get(i915);
1906 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1907 vlv_punit_put(i915);
1908 } else if (INTEL_GEN(i915) >= 6) {
1909 freq = intel_uncore_read(uncore, GEN6_RPSTAT1);
1910 } else {
1911 freq = intel_uncore_read(uncore, MEMSTAT_ILK);
1912 }
1913
1914 return intel_rps_get_cagf(rps, freq);
1915 }
1916
1917 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
1918 {
1919 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
1920 intel_wakeref_t wakeref;
1921 u32 freq = 0;
1922
1923 with_intel_runtime_pm_if_in_use(rpm, wakeref)
1924 freq = intel_gpu_freq(rps, read_cagf(rps));
1925
1926 return freq;
1927 }
1928
1929 /* External interface for intel_ips.ko */
1930
1931 static struct drm_i915_private __rcu *ips_mchdev;
1932
1933 /**
1934 * Tells the intel_ips driver that the i915 driver is now loaded, if
1935 * IPS got loaded first.
1936 *
1937 * This awkward dance is so that neither module has to depend on the
1938 * other in order for IPS to do the appropriate communication of
1939 * GPU turbo limits to i915.
1940 */
1941 static void
1942 ips_ping_for_i915_load(void)
1943 {
1944 void (*link)(void);
1945
1946 link = symbol_get(ips_link_to_i915_driver);
1947 if (link) {
1948 link();
1949 symbol_put(ips_link_to_i915_driver);
1950 }
1951 }
1952
1953 void intel_rps_driver_register(struct intel_rps *rps)
1954 {
1955 struct intel_gt *gt = rps_to_gt(rps);
1956
1957 /*
1958 * We only register the i915 ips part with intel-ips once everything is
1959 * set up, to avoid intel-ips sneaking in and reading bogus values.
1960 */
1961 if (IS_GEN(gt->i915, 5)) {
1962 GEM_BUG_ON(ips_mchdev);
1963 rcu_assign_pointer(ips_mchdev, gt->i915);
1964 ips_ping_for_i915_load();
1965 }
1966 }
1967
1968 void intel_rps_driver_unregister(struct intel_rps *rps)
1969 {
1970 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
1971 rcu_assign_pointer(ips_mchdev, NULL);
1972 }
1973
1974 static struct drm_i915_private *mchdev_get(void)
1975 {
1976 struct drm_i915_private *i915;
1977
1978 rcu_read_lock();
1979 i915 = rcu_dereference(ips_mchdev);
1980 if (i915 && !kref_get_unless_zero(&i915->drm.ref))
1981 i915 = NULL;
1982 rcu_read_unlock();
1983
1984 return i915;
1985 }
1986
1987 /**
1988 * i915_read_mch_val - return value for IPS use
1989 *
1990 * Calculate and return a value for the IPS driver to use when deciding whether
1991 * we have thermal and power headroom to increase CPU or GPU power budget.
1992 */
1993 unsigned long i915_read_mch_val(void)
1994 {
1995 struct drm_i915_private *i915;
1996 unsigned long chipset_val = 0;
1997 unsigned long graphics_val = 0;
1998 intel_wakeref_t wakeref;
1999
2000 i915 = mchdev_get();
2001 if (!i915)
2002 return 0;
2003
2004 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2005 struct intel_ips *ips = &i915->gt.rps.ips;
2006
2007 spin_lock_irq(&mchdev_lock);
2008 chipset_val = __ips_chipset_val(ips);
2009 graphics_val = __ips_gfx_val(ips);
2010 spin_unlock_irq(&mchdev_lock);
2011 }
2012
2013 drm_dev_put(&i915->drm);
2014 return chipset_val + graphics_val;
2015 }
2016 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2017
2018 /**
2019 * i915_gpu_raise - raise GPU frequency limit
2020 *
2021 * Raise the limit; IPS indicates we have thermal headroom.
2022 */
2023 bool i915_gpu_raise(void)
2024 {
2025 struct drm_i915_private *i915;
2026 struct intel_rps *rps;
2027
2028 i915 = mchdev_get();
2029 if (!i915)
2030 return false;
2031
2032 rps = &i915->gt.rps;
2033
2034 spin_lock_irq(&mchdev_lock);
2035 if (rps->max_freq_softlimit < rps->max_freq)
2036 rps->max_freq_softlimit++;
2037 spin_unlock_irq(&mchdev_lock);
2038
2039 drm_dev_put(&i915->drm);
2040 return true;
2041 }
2042 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2043
2044 /**
2045 * i915_gpu_lower - lower GPU frequency limit
2046 *
2047 * IPS indicates we're close to a thermal limit, so throttle back the GPU
2048 * frequency maximum.
2049 */
2050 bool i915_gpu_lower(void)
2051 {
2052 struct drm_i915_private *i915;
2053 struct intel_rps *rps;
2054
2055 i915 = mchdev_get();
2056 if (!i915)
2057 return false;
2058
2059 rps = &i915->gt.rps;
2060
2061 spin_lock_irq(&mchdev_lock);
2062 if (rps->max_freq_softlimit > rps->min_freq)
2063 rps->max_freq_softlimit--;
2064 spin_unlock_irq(&mchdev_lock);
2065
2066 drm_dev_put(&i915->drm);
2067 return true;
2068 }
2069 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2070
2071 /**
2072 * i915_gpu_busy - indicate GPU business to IPS
2073 *
2074 * Tell the IPS driver whether or not the GPU is busy.
2075 */
2076 bool i915_gpu_busy(void)
2077 {
2078 struct drm_i915_private *i915;
2079 bool ret;
2080
2081 i915 = mchdev_get();
2082 if (!i915)
2083 return false;
2084
2085 ret = i915->gt.awake;
2086
2087 drm_dev_put(&i915->drm);
2088 return ret;
2089 }
2090 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2091
2092 /**
2093 * i915_gpu_turbo_disable - disable graphics turbo
2094 *
2095 * Disable graphics turbo by resetting the max frequency and setting the
2096 * current frequency to the default.
2097 */
2098 bool i915_gpu_turbo_disable(void)
2099 {
2100 struct drm_i915_private *i915;
2101 struct intel_rps *rps;
2102 bool ret;
2103
2104 i915 = mchdev_get();
2105 if (!i915)
2106 return false;
2107
2108 rps = &i915->gt.rps;
2109
2110 spin_lock_irq(&mchdev_lock);
2111 rps->max_freq_softlimit = rps->min_freq;
2112 ret = gen5_rps_set(&i915->gt.rps, rps->min_freq);
2113 spin_unlock_irq(&mchdev_lock);
2114
2115 drm_dev_put(&i915->drm);
2116 return ret;
2117 }
2118 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2119
2120 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2121 #include "selftest_rps.c"
2122 #endif
Linux with added FPC-III support