?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/panel: panel-himax-hx83102: support for csot-pna957qt1-1 MIPI-DSI panel
[drm/drm-misc.git] / drivers / cxl / core / region.c
blobe8d11a988fd926a81e9eae993c2fe42d8e625787
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/genalloc.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/memory.h>
8 #include <linux/slab.h>
9 #include <linux/uuid.h>
10 #include <linux/sort.h>
11 #include <linux/idr.h>
12 #include <linux/memory-tiers.h>
13 #include <cxlmem.h>
14 #include <cxl.h>
15 #include "core.h"
16
17 /**
18 * DOC: cxl core region
19 *
20 * CXL Regions represent mapped memory capacity in system physical address
21 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
22 * Memory ranges, Regions represent the active mapped capacity by the HDM
23 * Decoder Capability structures throughout the Host Bridges, Switches, and
24 * Endpoints in the topology.
25 *
26 * Region configuration has ordering constraints. UUID may be set at any time
27 * but is only visible for persistent regions.
28 * 1. Interleave granularity
29 * 2. Interleave size
30 * 3. Decoder targets
31 */
32
33 static struct cxl_region *to_cxl_region(struct device *dev);
34
35 #define __ACCESS_ATTR_RO(_level, _name) { \
36 .attr = { .name = __stringify(_name), .mode = 0444 }, \
37 .show = _name##_access##_level##_show, \
38 }
39
40 #define ACCESS_DEVICE_ATTR_RO(level, name) \
41 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
42
43 #define ACCESS_ATTR_RO(level, attrib) \
44 static ssize_t attrib##_access##level##_show(struct device *dev, \
45 struct device_attribute *attr, \
46 char *buf) \
47 { \
48 struct cxl_region *cxlr = to_cxl_region(dev); \
49 \
50 if (cxlr->coord[level].attrib == 0) \
51 return -ENOENT; \
52 \
53 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \
54 } \
55 static ACCESS_DEVICE_ATTR_RO(level, attrib)
56
57 ACCESS_ATTR_RO(0, read_bandwidth);
58 ACCESS_ATTR_RO(0, read_latency);
59 ACCESS_ATTR_RO(0, write_bandwidth);
60 ACCESS_ATTR_RO(0, write_latency);
61
62 #define ACCESS_ATTR_DECLARE(level, attrib) \
63 (&dev_attr_access##level##_##attrib.attr)
64
65 static struct attribute *access0_coordinate_attrs[] = {
66 ACCESS_ATTR_DECLARE(0, read_bandwidth),
67 ACCESS_ATTR_DECLARE(0, write_bandwidth),
68 ACCESS_ATTR_DECLARE(0, read_latency),
69 ACCESS_ATTR_DECLARE(0, write_latency),
70 NULL
71 };
72
73 ACCESS_ATTR_RO(1, read_bandwidth);
74 ACCESS_ATTR_RO(1, read_latency);
75 ACCESS_ATTR_RO(1, write_bandwidth);
76 ACCESS_ATTR_RO(1, write_latency);
77
78 static struct attribute *access1_coordinate_attrs[] = {
79 ACCESS_ATTR_DECLARE(1, read_bandwidth),
80 ACCESS_ATTR_DECLARE(1, write_bandwidth),
81 ACCESS_ATTR_DECLARE(1, read_latency),
82 ACCESS_ATTR_DECLARE(1, write_latency),
83 NULL
84 };
85
86 #define ACCESS_VISIBLE(level) \
87 static umode_t cxl_region_access##level##_coordinate_visible( \
88 struct kobject *kobj, struct attribute *a, int n) \
89 { \
90 struct device *dev = kobj_to_dev(kobj); \
91 struct cxl_region *cxlr = to_cxl_region(dev); \
92 \
93 if (a == &dev_attr_access##level##_read_latency.attr && \
94 cxlr->coord[level].read_latency == 0) \
95 return 0; \
96 \
97 if (a == &dev_attr_access##level##_write_latency.attr && \
98 cxlr->coord[level].write_latency == 0) \
99 return 0; \
100 \
101 if (a == &dev_attr_access##level##_read_bandwidth.attr && \
102 cxlr->coord[level].read_bandwidth == 0) \
103 return 0; \
104 \
105 if (a == &dev_attr_access##level##_write_bandwidth.attr && \
106 cxlr->coord[level].write_bandwidth == 0) \
107 return 0; \
108 \
109 return a->mode; \
110 }
111
112 ACCESS_VISIBLE(0);
113 ACCESS_VISIBLE(1);
114
115 static const struct attribute_group cxl_region_access0_coordinate_group = {
116 .name = "access0",
117 .attrs = access0_coordinate_attrs,
118 .is_visible = cxl_region_access0_coordinate_visible,
119 };
120
121 static const struct attribute_group *get_cxl_region_access0_group(void)
122 {
123 return &cxl_region_access0_coordinate_group;
124 }
125
126 static const struct attribute_group cxl_region_access1_coordinate_group = {
127 .name = "access1",
128 .attrs = access1_coordinate_attrs,
129 .is_visible = cxl_region_access1_coordinate_visible,
130 };
131
132 static const struct attribute_group *get_cxl_region_access1_group(void)
133 {
134 return &cxl_region_access1_coordinate_group;
135 }
136
137 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
138 char *buf)
139 {
140 struct cxl_region *cxlr = to_cxl_region(dev);
141 struct cxl_region_params *p = &cxlr->params;
142 ssize_t rc;
143
144 rc = down_read_interruptible(&cxl_region_rwsem);
145 if (rc)
146 return rc;
147 if (cxlr->mode != CXL_DECODER_PMEM)
148 rc = sysfs_emit(buf, "\n");
149 else
150 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
151 up_read(&cxl_region_rwsem);
152
153 return rc;
154 }
155
156 static int is_dup(struct device *match, void *data)
157 {
158 struct cxl_region_params *p;
159 struct cxl_region *cxlr;
160 uuid_t *uuid = data;
161
162 if (!is_cxl_region(match))
163 return 0;
164
165 lockdep_assert_held(&cxl_region_rwsem);
166 cxlr = to_cxl_region(match);
167 p = &cxlr->params;
168
169 if (uuid_equal(&p->uuid, uuid)) {
170 dev_dbg(match, "already has uuid: %pUb\n", uuid);
171 return -EBUSY;
172 }
173
174 return 0;
175 }
176
177 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
178 const char *buf, size_t len)
179 {
180 struct cxl_region *cxlr = to_cxl_region(dev);
181 struct cxl_region_params *p = &cxlr->params;
182 uuid_t temp;
183 ssize_t rc;
184
185 if (len != UUID_STRING_LEN + 1)
186 return -EINVAL;
187
188 rc = uuid_parse(buf, &temp);
189 if (rc)
190 return rc;
191
192 if (uuid_is_null(&temp))
193 return -EINVAL;
194
195 rc = down_write_killable(&cxl_region_rwsem);
196 if (rc)
197 return rc;
198
199 if (uuid_equal(&p->uuid, &temp))
200 goto out;
201
202 rc = -EBUSY;
203 if (p->state >= CXL_CONFIG_ACTIVE)
204 goto out;
205
206 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
207 if (rc < 0)
208 goto out;
209
210 uuid_copy(&p->uuid, &temp);
211 out:
212 up_write(&cxl_region_rwsem);
213
214 if (rc)
215 return rc;
216 return len;
217 }
218 static DEVICE_ATTR_RW(uuid);
219
220 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
221 struct cxl_region *cxlr)
222 {
223 return xa_load(&port->regions, (unsigned long)cxlr);
224 }
225
226 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
227 {
228 if (!cpu_cache_has_invalidate_memregion()) {
229 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
230 dev_info_once(
231 &cxlr->dev,
232 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
233 return 0;
234 } else {
235 dev_WARN(&cxlr->dev,
236 "Failed to synchronize CPU cache state\n");
237 return -ENXIO;
238 }
239 }
240
241 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
242 return 0;
243 }
244
245 static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
246 {
247 struct cxl_region_params *p = &cxlr->params;
248 int i;
249
250 /*
251 * Before region teardown attempt to flush, evict any data cached for
252 * this region, or scream loudly about missing arch / platform support
253 * for CXL teardown.
254 */
255 cxl_region_invalidate_memregion(cxlr);
256
257 for (i = count - 1; i >= 0; i--) {
258 struct cxl_endpoint_decoder *cxled = p->targets[i];
259 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
260 struct cxl_port *iter = cxled_to_port(cxled);
261 struct cxl_dev_state *cxlds = cxlmd->cxlds;
262 struct cxl_ep *ep;
263
264 if (cxlds->rcd)
265 goto endpoint_reset;
266
267 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
268 iter = to_cxl_port(iter->dev.parent);
269
270 for (ep = cxl_ep_load(iter, cxlmd); iter;
271 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
272 struct cxl_region_ref *cxl_rr;
273 struct cxl_decoder *cxld;
274
275 cxl_rr = cxl_rr_load(iter, cxlr);
276 cxld = cxl_rr->decoder;
277 if (cxld->reset)
278 cxld->reset(cxld);
279 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
280 }
281
282 endpoint_reset:
283 cxled->cxld.reset(&cxled->cxld);
284 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
285 }
286
287 /* all decoders associated with this region have been torn down */
288 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
289 }
290
291 static int commit_decoder(struct cxl_decoder *cxld)
292 {
293 struct cxl_switch_decoder *cxlsd = NULL;
294
295 if (cxld->commit)
296 return cxld->commit(cxld);
297
298 if (is_switch_decoder(&cxld->dev))
299 cxlsd = to_cxl_switch_decoder(&cxld->dev);
300
301 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
302 "->commit() is required\n"))
303 return -ENXIO;
304 return 0;
305 }
306
307 static int cxl_region_decode_commit(struct cxl_region *cxlr)
308 {
309 struct cxl_region_params *p = &cxlr->params;
310 int i, rc = 0;
311
312 for (i = 0; i < p->nr_targets; i++) {
313 struct cxl_endpoint_decoder *cxled = p->targets[i];
314 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
315 struct cxl_region_ref *cxl_rr;
316 struct cxl_decoder *cxld;
317 struct cxl_port *iter;
318 struct cxl_ep *ep;
319
320 /* commit bottom up */
321 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
322 iter = to_cxl_port(iter->dev.parent)) {
323 cxl_rr = cxl_rr_load(iter, cxlr);
324 cxld = cxl_rr->decoder;
325 rc = commit_decoder(cxld);
326 if (rc)
327 break;
328 }
329
330 if (rc) {
331 /* programming @iter failed, teardown */
332 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
333 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
334 cxl_rr = cxl_rr_load(iter, cxlr);
335 cxld = cxl_rr->decoder;
336 if (cxld->reset)
337 cxld->reset(cxld);
338 }
339
340 cxled->cxld.reset(&cxled->cxld);
341 goto err;
342 }
343 }
344
345 return 0;
346
347 err:
348 /* undo the targets that were successfully committed */
349 cxl_region_decode_reset(cxlr, i);
350 return rc;
351 }
352
353 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
354 const char *buf, size_t len)
355 {
356 struct cxl_region *cxlr = to_cxl_region(dev);
357 struct cxl_region_params *p = &cxlr->params;
358 bool commit;
359 ssize_t rc;
360
361 rc = kstrtobool(buf, &commit);
362 if (rc)
363 return rc;
364
365 rc = down_write_killable(&cxl_region_rwsem);
366 if (rc)
367 return rc;
368
369 /* Already in the requested state? */
370 if (commit && p->state >= CXL_CONFIG_COMMIT)
371 goto out;
372 if (!commit && p->state < CXL_CONFIG_COMMIT)
373 goto out;
374
375 /* Not ready to commit? */
376 if (commit && p->state < CXL_CONFIG_ACTIVE) {
377 rc = -ENXIO;
378 goto out;
379 }
380
381 /*
382 * Invalidate caches before region setup to drop any speculative
383 * consumption of this address space
384 */
385 rc = cxl_region_invalidate_memregion(cxlr);
386 if (rc)
387 goto out;
388
389 if (commit) {
390 rc = cxl_region_decode_commit(cxlr);
391 if (rc == 0)
392 p->state = CXL_CONFIG_COMMIT;
393 } else {
394 p->state = CXL_CONFIG_RESET_PENDING;
395 up_write(&cxl_region_rwsem);
396 device_release_driver(&cxlr->dev);
397 down_write(&cxl_region_rwsem);
398
399 /*
400 * The lock was dropped, so need to revalidate that the reset is
401 * still pending.
402 */
403 if (p->state == CXL_CONFIG_RESET_PENDING) {
404 cxl_region_decode_reset(cxlr, p->interleave_ways);
405 p->state = CXL_CONFIG_ACTIVE;
406 }
407 }
408
409 out:
410 up_write(&cxl_region_rwsem);
411
412 if (rc)
413 return rc;
414 return len;
415 }
416
417 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
418 char *buf)
419 {
420 struct cxl_region *cxlr = to_cxl_region(dev);
421 struct cxl_region_params *p = &cxlr->params;
422 ssize_t rc;
423
424 rc = down_read_interruptible(&cxl_region_rwsem);
425 if (rc)
426 return rc;
427 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
428 up_read(&cxl_region_rwsem);
429
430 return rc;
431 }
432 static DEVICE_ATTR_RW(commit);
433
434 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
435 int n)
436 {
437 struct device *dev = kobj_to_dev(kobj);
438 struct cxl_region *cxlr = to_cxl_region(dev);
439
440 /*
441 * Support tooling that expects to find a 'uuid' attribute for all
442 * regions regardless of mode.
443 */
444 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
445 return 0444;
446 return a->mode;
447 }
448
449 static ssize_t interleave_ways_show(struct device *dev,
450 struct device_attribute *attr, char *buf)
451 {
452 struct cxl_region *cxlr = to_cxl_region(dev);
453 struct cxl_region_params *p = &cxlr->params;
454 ssize_t rc;
455
456 rc = down_read_interruptible(&cxl_region_rwsem);
457 if (rc)
458 return rc;
459 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
460 up_read(&cxl_region_rwsem);
461
462 return rc;
463 }
464
465 static const struct attribute_group *get_cxl_region_target_group(void);
466
467 static ssize_t interleave_ways_store(struct device *dev,
468 struct device_attribute *attr,
469 const char *buf, size_t len)
470 {
471 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
472 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
473 struct cxl_region *cxlr = to_cxl_region(dev);
474 struct cxl_region_params *p = &cxlr->params;
475 unsigned int val, save;
476 int rc;
477 u8 iw;
478
479 rc = kstrtouint(buf, 0, &val);
480 if (rc)
481 return rc;
482
483 rc = ways_to_eiw(val, &iw);
484 if (rc)
485 return rc;
486
487 /*
488 * Even for x3, x6, and x12 interleaves the region interleave must be a
489 * power of 2 multiple of the host bridge interleave.
490 */
491 if (!is_power_of_2(val / cxld->interleave_ways) ||
492 (val % cxld->interleave_ways)) {
493 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
494 return -EINVAL;
495 }
496
497 rc = down_write_killable(&cxl_region_rwsem);
498 if (rc)
499 return rc;
500 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
501 rc = -EBUSY;
502 goto out;
503 }
504
505 save = p->interleave_ways;
506 p->interleave_ways = val;
507 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
508 if (rc)
509 p->interleave_ways = save;
510 out:
511 up_write(&cxl_region_rwsem);
512 if (rc)
513 return rc;
514 return len;
515 }
516 static DEVICE_ATTR_RW(interleave_ways);
517
518 static ssize_t interleave_granularity_show(struct device *dev,
519 struct device_attribute *attr,
520 char *buf)
521 {
522 struct cxl_region *cxlr = to_cxl_region(dev);
523 struct cxl_region_params *p = &cxlr->params;
524 ssize_t rc;
525
526 rc = down_read_interruptible(&cxl_region_rwsem);
527 if (rc)
528 return rc;
529 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
530 up_read(&cxl_region_rwsem);
531
532 return rc;
533 }
534
535 static ssize_t interleave_granularity_store(struct device *dev,
536 struct device_attribute *attr,
537 const char *buf, size_t len)
538 {
539 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
540 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
541 struct cxl_region *cxlr = to_cxl_region(dev);
542 struct cxl_region_params *p = &cxlr->params;
543 int rc, val;
544 u16 ig;
545
546 rc = kstrtoint(buf, 0, &val);
547 if (rc)
548 return rc;
549
550 rc = granularity_to_eig(val, &ig);
551 if (rc)
552 return rc;
553
554 /*
555 * When the host-bridge is interleaved, disallow region granularity !=
556 * root granularity. Regions with a granularity less than the root
557 * interleave result in needing multiple endpoints to support a single
558 * slot in the interleave (possible to support in the future). Regions
559 * with a granularity greater than the root interleave result in invalid
560 * DPA translations (invalid to support).
561 */
562 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
563 return -EINVAL;
564
565 rc = down_write_killable(&cxl_region_rwsem);
566 if (rc)
567 return rc;
568 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
569 rc = -EBUSY;
570 goto out;
571 }
572
573 p->interleave_granularity = val;
574 out:
575 up_write(&cxl_region_rwsem);
576 if (rc)
577 return rc;
578 return len;
579 }
580 static DEVICE_ATTR_RW(interleave_granularity);
581
582 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
583 char *buf)
584 {
585 struct cxl_region *cxlr = to_cxl_region(dev);
586 struct cxl_region_params *p = &cxlr->params;
587 u64 resource = -1ULL;
588 ssize_t rc;
589
590 rc = down_read_interruptible(&cxl_region_rwsem);
591 if (rc)
592 return rc;
593 if (p->res)
594 resource = p->res->start;
595 rc = sysfs_emit(buf, "%#llx\n", resource);
596 up_read(&cxl_region_rwsem);
597
598 return rc;
599 }
600 static DEVICE_ATTR_RO(resource);
601
602 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
603 char *buf)
604 {
605 struct cxl_region *cxlr = to_cxl_region(dev);
606
607 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
608 }
609 static DEVICE_ATTR_RO(mode);
610
611 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
612 {
613 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
614 struct cxl_region_params *p = &cxlr->params;
615 struct resource *res;
616 u64 remainder = 0;
617
618 lockdep_assert_held_write(&cxl_region_rwsem);
619
620 /* Nothing to do... */
621 if (p->res && resource_size(p->res) == size)
622 return 0;
623
624 /* To change size the old size must be freed first */
625 if (p->res)
626 return -EBUSY;
627
628 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
629 return -EBUSY;
630
631 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
632 if (!p->interleave_ways || !p->interleave_granularity ||
633 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
634 return -ENXIO;
635
636 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
637 if (remainder)
638 return -EINVAL;
639
640 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
641 dev_name(&cxlr->dev));
642 if (IS_ERR(res)) {
643 dev_dbg(&cxlr->dev,
644 "HPA allocation error (%ld) for size:%pap in %s %pr\n",
645 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
646 return PTR_ERR(res);
647 }
648
649 p->res = res;
650 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
651
652 return 0;
653 }
654
655 static void cxl_region_iomem_release(struct cxl_region *cxlr)
656 {
657 struct cxl_region_params *p = &cxlr->params;
658
659 if (device_is_registered(&cxlr->dev))
660 lockdep_assert_held_write(&cxl_region_rwsem);
661 if (p->res) {
662 /*
663 * Autodiscovered regions may not have been able to insert their
664 * resource.
665 */
666 if (p->res->parent)
667 remove_resource(p->res);
668 kfree(p->res);
669 p->res = NULL;
670 }
671 }
672
673 static int free_hpa(struct cxl_region *cxlr)
674 {
675 struct cxl_region_params *p = &cxlr->params;
676
677 lockdep_assert_held_write(&cxl_region_rwsem);
678
679 if (!p->res)
680 return 0;
681
682 if (p->state >= CXL_CONFIG_ACTIVE)
683 return -EBUSY;
684
685 cxl_region_iomem_release(cxlr);
686 p->state = CXL_CONFIG_IDLE;
687 return 0;
688 }
689
690 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
691 const char *buf, size_t len)
692 {
693 struct cxl_region *cxlr = to_cxl_region(dev);
694 u64 val;
695 int rc;
696
697 rc = kstrtou64(buf, 0, &val);
698 if (rc)
699 return rc;
700
701 rc = down_write_killable(&cxl_region_rwsem);
702 if (rc)
703 return rc;
704
705 if (val)
706 rc = alloc_hpa(cxlr, val);
707 else
708 rc = free_hpa(cxlr);
709 up_write(&cxl_region_rwsem);
710
711 if (rc)
712 return rc;
713
714 return len;
715 }
716
717 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
718 char *buf)
719 {
720 struct cxl_region *cxlr = to_cxl_region(dev);
721 struct cxl_region_params *p = &cxlr->params;
722 u64 size = 0;
723 ssize_t rc;
724
725 rc = down_read_interruptible(&cxl_region_rwsem);
726 if (rc)
727 return rc;
728 if (p->res)
729 size = resource_size(p->res);
730 rc = sysfs_emit(buf, "%#llx\n", size);
731 up_read(&cxl_region_rwsem);
732
733 return rc;
734 }
735 static DEVICE_ATTR_RW(size);
736
737 static struct attribute *cxl_region_attrs[] = {
738 &dev_attr_uuid.attr,
739 &dev_attr_commit.attr,
740 &dev_attr_interleave_ways.attr,
741 &dev_attr_interleave_granularity.attr,
742 &dev_attr_resource.attr,
743 &dev_attr_size.attr,
744 &dev_attr_mode.attr,
745 NULL,
746 };
747
748 static const struct attribute_group cxl_region_group = {
749 .attrs = cxl_region_attrs,
750 .is_visible = cxl_region_visible,
751 };
752
753 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
754 {
755 struct cxl_region_params *p = &cxlr->params;
756 struct cxl_endpoint_decoder *cxled;
757 int rc;
758
759 rc = down_read_interruptible(&cxl_region_rwsem);
760 if (rc)
761 return rc;
762
763 if (pos >= p->interleave_ways) {
764 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
765 p->interleave_ways);
766 rc = -ENXIO;
767 goto out;
768 }
769
770 cxled = p->targets[pos];
771 if (!cxled)
772 rc = sysfs_emit(buf, "\n");
773 else
774 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
775 out:
776 up_read(&cxl_region_rwsem);
777
778 return rc;
779 }
780
781 static int check_commit_order(struct device *dev, void *data)
782 {
783 struct cxl_decoder *cxld = to_cxl_decoder(dev);
784
785 /*
786 * if port->commit_end is not the only free decoder, then out of
787 * order shutdown has occurred, block further allocations until
788 * that is resolved
789 */
790 if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0))
791 return -EBUSY;
792 return 0;
793 }
794
795 static int match_free_decoder(struct device *dev, const void *data)
796 {
797 struct cxl_port *port = to_cxl_port(dev->parent);
798 struct cxl_decoder *cxld;
799 int rc;
800
801 if (!is_switch_decoder(dev))
802 return 0;
803
804 cxld = to_cxl_decoder(dev);
805
806 if (cxld->id != port->commit_end + 1)
807 return 0;
808
809 if (cxld->region) {
810 dev_dbg(dev->parent,
811 "next decoder to commit (%s) is already reserved (%s)\n",
812 dev_name(dev), dev_name(&cxld->region->dev));
813 return 0;
814 }
815
816 rc = device_for_each_child_reverse_from(dev->parent, dev, NULL,
817 check_commit_order);
818 if (rc) {
819 dev_dbg(dev->parent,
820 "unable to allocate %s due to out of order shutdown\n",
821 dev_name(dev));
822 return 0;
823 }
824 return 1;
825 }
826
827 static int match_auto_decoder(struct device *dev, const void *data)
828 {
829 const struct cxl_region_params *p = data;
830 struct cxl_decoder *cxld;
831 struct range *r;
832
833 if (!is_switch_decoder(dev))
834 return 0;
835
836 cxld = to_cxl_decoder(dev);
837 r = &cxld->hpa_range;
838
839 if (p->res && p->res->start == r->start && p->res->end == r->end)
840 return 1;
841
842 return 0;
843 }
844
845 static struct cxl_decoder *
846 cxl_region_find_decoder(struct cxl_port *port,
847 struct cxl_endpoint_decoder *cxled,
848 struct cxl_region *cxlr)
849 {
850 struct device *dev;
851
852 if (port == cxled_to_port(cxled))
853 return &cxled->cxld;
854
855 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
856 dev = device_find_child(&port->dev, &cxlr->params,
857 match_auto_decoder);
858 else
859 dev = device_find_child(&port->dev, NULL, match_free_decoder);
860 if (!dev)
861 return NULL;
862 /*
863 * This decoder is pinned registered as long as the endpoint decoder is
864 * registered, and endpoint decoder unregistration holds the
865 * cxl_region_rwsem over unregister events, so no need to hold on to
866 * this extra reference.
867 */
868 put_device(dev);
869 return to_cxl_decoder(dev);
870 }
871
872 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
873 struct cxl_decoder *cxld)
874 {
875 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
876 struct cxl_decoder *cxld_iter = rr->decoder;
877
878 /*
879 * Allow the out of order assembly of auto-discovered regions.
880 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
881 * in HPA order. Confirm that the decoder with the lesser HPA
882 * starting address has the lesser id.
883 */
884 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
885 dev_name(&cxld->dev), cxld->id,
886 dev_name(&cxld_iter->dev), cxld_iter->id);
887
888 if (cxld_iter->id > cxld->id)
889 return true;
890
891 return false;
892 }
893
894 static struct cxl_region_ref *
895 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
896 struct cxl_endpoint_decoder *cxled)
897 {
898 struct cxl_region_params *p = &cxlr->params;
899 struct cxl_region_ref *cxl_rr, *iter;
900 unsigned long index;
901 int rc;
902
903 xa_for_each(&port->regions, index, iter) {
904 struct cxl_region_params *ip = &iter->region->params;
905
906 if (!ip->res || ip->res->start < p->res->start)
907 continue;
908
909 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
910 struct cxl_decoder *cxld;
911
912 cxld = cxl_region_find_decoder(port, cxled, cxlr);
913 if (auto_order_ok(port, iter->region, cxld))
914 continue;
915 }
916 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
917 dev_name(&port->dev),
918 dev_name(&iter->region->dev), ip->res, p->res);
919
920 return ERR_PTR(-EBUSY);
921 }
922
923 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
924 if (!cxl_rr)
925 return ERR_PTR(-ENOMEM);
926 cxl_rr->port = port;
927 cxl_rr->region = cxlr;
928 cxl_rr->nr_targets = 1;
929 xa_init(&cxl_rr->endpoints);
930
931 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
932 if (rc) {
933 dev_dbg(&cxlr->dev,
934 "%s: failed to track region reference: %d\n",
935 dev_name(&port->dev), rc);
936 kfree(cxl_rr);
937 return ERR_PTR(rc);
938 }
939
940 return cxl_rr;
941 }
942
943 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
944 {
945 struct cxl_region *cxlr = cxl_rr->region;
946 struct cxl_decoder *cxld = cxl_rr->decoder;
947
948 if (!cxld)
949 return;
950
951 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
952 if (cxld->region == cxlr) {
953 cxld->region = NULL;
954 put_device(&cxlr->dev);
955 }
956 }
957
958 static void free_region_ref(struct cxl_region_ref *cxl_rr)
959 {
960 struct cxl_port *port = cxl_rr->port;
961 struct cxl_region *cxlr = cxl_rr->region;
962
963 cxl_rr_free_decoder(cxl_rr);
964 xa_erase(&port->regions, (unsigned long)cxlr);
965 xa_destroy(&cxl_rr->endpoints);
966 kfree(cxl_rr);
967 }
968
969 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
970 struct cxl_endpoint_decoder *cxled)
971 {
972 int rc;
973 struct cxl_port *port = cxl_rr->port;
974 struct cxl_region *cxlr = cxl_rr->region;
975 struct cxl_decoder *cxld = cxl_rr->decoder;
976 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
977
978 if (ep) {
979 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
980 GFP_KERNEL);
981 if (rc)
982 return rc;
983 }
984 cxl_rr->nr_eps++;
985
986 if (!cxld->region) {
987 cxld->region = cxlr;
988 get_device(&cxlr->dev);
989 }
990
991 return 0;
992 }
993
994 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
995 struct cxl_endpoint_decoder *cxled,
996 struct cxl_region_ref *cxl_rr)
997 {
998 struct cxl_decoder *cxld;
999
1000 cxld = cxl_region_find_decoder(port, cxled, cxlr);
1001 if (!cxld) {
1002 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
1003 dev_name(&port->dev));
1004 return -EBUSY;
1005 }
1006
1007 if (cxld->region) {
1008 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1009 dev_name(&port->dev), dev_name(&cxld->dev),
1010 dev_name(&cxld->region->dev));
1011 return -EBUSY;
1012 }
1013
1014 /*
1015 * Endpoints should already match the region type, but backstop that
1016 * assumption with an assertion. Switch-decoders change mapping-type
1017 * based on what is mapped when they are assigned to a region.
1018 */
1019 dev_WARN_ONCE(&cxlr->dev,
1020 port == cxled_to_port(cxled) &&
1021 cxld->target_type != cxlr->type,
1022 "%s:%s mismatch decoder type %d -> %d\n",
1023 dev_name(&cxled_to_memdev(cxled)->dev),
1024 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1025 cxld->target_type = cxlr->type;
1026 cxl_rr->decoder = cxld;
1027 return 0;
1028 }
1029
1030 /**
1031 * cxl_port_attach_region() - track a region's interest in a port by endpoint
1032 * @port: port to add a new region reference 'struct cxl_region_ref'
1033 * @cxlr: region to attach to @port
1034 * @cxled: endpoint decoder used to create or further pin a region reference
1035 * @pos: interleave position of @cxled in @cxlr
1036 *
1037 * The attach event is an opportunity to validate CXL decode setup
1038 * constraints and record metadata needed for programming HDM decoders,
1039 * in particular decoder target lists.
1040 *
1041 * The steps are:
1042 *
1043 * - validate that there are no other regions with a higher HPA already
1044 * associated with @port
1045 * - establish a region reference if one is not already present
1046 *
1047 * - additionally allocate a decoder instance that will host @cxlr on
1048 * @port
1049 *
1050 * - pin the region reference by the endpoint
1051 * - account for how many entries in @port's target list are needed to
1052 * cover all of the added endpoints.
1053 */
1054 static int cxl_port_attach_region(struct cxl_port *port,
1055 struct cxl_region *cxlr,
1056 struct cxl_endpoint_decoder *cxled, int pos)
1057 {
1058 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1059 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1060 struct cxl_region_ref *cxl_rr;
1061 bool nr_targets_inc = false;
1062 struct cxl_decoder *cxld;
1063 unsigned long index;
1064 int rc = -EBUSY;
1065
1066 lockdep_assert_held_write(&cxl_region_rwsem);
1067
1068 cxl_rr = cxl_rr_load(port, cxlr);
1069 if (cxl_rr) {
1070 struct cxl_ep *ep_iter;
1071 int found = 0;
1072
1073 /*
1074 * Walk the existing endpoints that have been attached to
1075 * @cxlr at @port and see if they share the same 'next' port
1076 * in the downstream direction. I.e. endpoints that share common
1077 * upstream switch.
1078 */
1079 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1080 if (ep_iter == ep)
1081 continue;
1082 if (ep_iter->next == ep->next) {
1083 found++;
1084 break;
1085 }
1086 }
1087
1088 /*
1089 * New target port, or @port is an endpoint port that always
1090 * accounts its own local decode as a target.
1091 */
1092 if (!found || !ep->next) {
1093 cxl_rr->nr_targets++;
1094 nr_targets_inc = true;
1095 }
1096 } else {
1097 cxl_rr = alloc_region_ref(port, cxlr, cxled);
1098 if (IS_ERR(cxl_rr)) {
1099 dev_dbg(&cxlr->dev,
1100 "%s: failed to allocate region reference\n",
1101 dev_name(&port->dev));
1102 return PTR_ERR(cxl_rr);
1103 }
1104 nr_targets_inc = true;
1105
1106 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
1107 if (rc)
1108 goto out_erase;
1109 }
1110 cxld = cxl_rr->decoder;
1111
1112 /*
1113 * the number of targets should not exceed the target_count
1114 * of the decoder
1115 */
1116 if (is_switch_decoder(&cxld->dev)) {
1117 struct cxl_switch_decoder *cxlsd;
1118
1119 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1120 if (cxl_rr->nr_targets > cxlsd->nr_targets) {
1121 dev_dbg(&cxlr->dev,
1122 "%s:%s %s add: %s:%s @ %d overflows targets: %d\n",
1123 dev_name(port->uport_dev), dev_name(&port->dev),
1124 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1125 dev_name(&cxled->cxld.dev), pos,
1126 cxlsd->nr_targets);
1127 rc = -ENXIO;
1128 goto out_erase;
1129 }
1130 }
1131
1132 rc = cxl_rr_ep_add(cxl_rr, cxled);
1133 if (rc) {
1134 dev_dbg(&cxlr->dev,
1135 "%s: failed to track endpoint %s:%s reference\n",
1136 dev_name(&port->dev), dev_name(&cxlmd->dev),
1137 dev_name(&cxld->dev));
1138 goto out_erase;
1139 }
1140
1141 dev_dbg(&cxlr->dev,
1142 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1143 dev_name(port->uport_dev), dev_name(&port->dev),
1144 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1145 dev_name(&cxled->cxld.dev), pos,
1146 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1147 dev_name(&cxlmd->dev) :
1148 "none",
1149 cxl_rr->nr_eps, cxl_rr->nr_targets);
1150
1151 return 0;
1152 out_erase:
1153 if (nr_targets_inc)
1154 cxl_rr->nr_targets--;
1155 if (cxl_rr->nr_eps == 0)
1156 free_region_ref(cxl_rr);
1157 return rc;
1158 }
1159
1160 static void cxl_port_detach_region(struct cxl_port *port,
1161 struct cxl_region *cxlr,
1162 struct cxl_endpoint_decoder *cxled)
1163 {
1164 struct cxl_region_ref *cxl_rr;
1165 struct cxl_ep *ep = NULL;
1166
1167 lockdep_assert_held_write(&cxl_region_rwsem);
1168
1169 cxl_rr = cxl_rr_load(port, cxlr);
1170 if (!cxl_rr)
1171 return;
1172
1173 /*
1174 * Endpoint ports do not carry cxl_ep references, and they
1175 * never target more than one endpoint by definition
1176 */
1177 if (cxl_rr->decoder == &cxled->cxld)
1178 cxl_rr->nr_eps--;
1179 else
1180 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1181 if (ep) {
1182 struct cxl_ep *ep_iter;
1183 unsigned long index;
1184 int found = 0;
1185
1186 cxl_rr->nr_eps--;
1187 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1188 if (ep_iter->next == ep->next) {
1189 found++;
1190 break;
1191 }
1192 }
1193 if (!found)
1194 cxl_rr->nr_targets--;
1195 }
1196
1197 if (cxl_rr->nr_eps == 0)
1198 free_region_ref(cxl_rr);
1199 }
1200
1201 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1202 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1203 int distance)
1204 {
1205 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1206 struct cxl_region *cxlr = cxl_rr->region;
1207 struct cxl_region_params *p = &cxlr->params;
1208 struct cxl_endpoint_decoder *cxled_peer;
1209 struct cxl_port *port = cxl_rr->port;
1210 struct cxl_memdev *cxlmd_peer;
1211 struct cxl_ep *ep_peer;
1212 int pos = cxled->pos;
1213
1214 /*
1215 * If this position wants to share a dport with the last endpoint mapped
1216 * then that endpoint, at index 'position - distance', must also be
1217 * mapped by this dport.
1218 */
1219 if (pos < distance) {
1220 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1221 dev_name(port->uport_dev), dev_name(&port->dev),
1222 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1223 return -ENXIO;
1224 }
1225 cxled_peer = p->targets[pos - distance];
1226 cxlmd_peer = cxled_to_memdev(cxled_peer);
1227 ep_peer = cxl_ep_load(port, cxlmd_peer);
1228 if (ep->dport != ep_peer->dport) {
1229 dev_dbg(&cxlr->dev,
1230 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1231 dev_name(port->uport_dev), dev_name(&port->dev),
1232 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1233 dev_name(&cxlmd_peer->dev),
1234 dev_name(&cxled_peer->cxld.dev));
1235 return -ENXIO;
1236 }
1237
1238 return 0;
1239 }
1240
1241 static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
1242 {
1243 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
1244 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
1245 unsigned int interleave_mask;
1246 u8 eiw;
1247 u16 eig;
1248 int high_pos, low_pos;
1249
1250 if (!test_bit(iw, &cxlhdm->iw_cap_mask))
1251 return -ENXIO;
1252 /*
1253 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
1254 * if eiw < 8:
1255 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
1256 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1257 *
1258 * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
1259 * interleave bits are none.
1260 *
1261 * if eiw >= 8:
1262 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
1263 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1264 *
1265 * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
1266 * interleave bits are none.
1267 */
1268 ways_to_eiw(iw, &eiw);
1269 if (eiw == 0 || eiw == 8)
1270 return 0;
1271
1272 granularity_to_eig(ig, &eig);
1273 if (eiw > 8)
1274 high_pos = eiw + eig - 1;
1275 else
1276 high_pos = eiw + eig + 7;
1277 low_pos = eig + 8;
1278 interleave_mask = GENMASK(high_pos, low_pos);
1279 if (interleave_mask & ~cxlhdm->interleave_mask)
1280 return -ENXIO;
1281
1282 return 0;
1283 }
1284
1285 static int cxl_port_setup_targets(struct cxl_port *port,
1286 struct cxl_region *cxlr,
1287 struct cxl_endpoint_decoder *cxled)
1288 {
1289 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1290 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1291 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1292 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1293 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1294 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1295 struct cxl_region_params *p = &cxlr->params;
1296 struct cxl_decoder *cxld = cxl_rr->decoder;
1297 struct cxl_switch_decoder *cxlsd;
1298 struct cxl_port *iter = port;
1299 u16 eig, peig;
1300 u8 eiw, peiw;
1301
1302 /*
1303 * While root level decoders support x3, x6, x12, switch level
1304 * decoders only support powers of 2 up to x16.
1305 */
1306 if (!is_power_of_2(cxl_rr->nr_targets)) {
1307 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1308 dev_name(port->uport_dev), dev_name(&port->dev),
1309 cxl_rr->nr_targets);
1310 return -EINVAL;
1311 }
1312
1313 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1314 if (cxl_rr->nr_targets_set) {
1315 int i, distance = 1;
1316 struct cxl_region_ref *cxl_rr_iter;
1317
1318 /*
1319 * The "distance" between peer downstream ports represents which
1320 * endpoint positions in the region interleave a given port can
1321 * host.
1322 *
1323 * For example, at the root of a hierarchy the distance is
1324 * always 1 as every index targets a different host-bridge. At
1325 * each subsequent switch level those ports map every Nth region
1326 * position where N is the width of the switch == distance.
1327 */
1328 do {
1329 cxl_rr_iter = cxl_rr_load(iter, cxlr);
1330 distance *= cxl_rr_iter->nr_targets;
1331 iter = to_cxl_port(iter->dev.parent);
1332 } while (!is_cxl_root(iter));
1333 distance *= cxlrd->cxlsd.cxld.interleave_ways;
1334
1335 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1336 if (ep->dport == cxlsd->target[i]) {
1337 rc = check_last_peer(cxled, ep, cxl_rr,
1338 distance);
1339 if (rc)
1340 return rc;
1341 goto out_target_set;
1342 }
1343 goto add_target;
1344 }
1345
1346 if (is_cxl_root(parent_port)) {
1347 /*
1348 * Root decoder IG is always set to value in CFMWS which
1349 * may be different than this region's IG. We can use the
1350 * region's IG here since interleave_granularity_store()
1351 * does not allow interleaved host-bridges with
1352 * root IG != region IG.
1353 */
1354 parent_ig = p->interleave_granularity;
1355 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1356 /*
1357 * For purposes of address bit routing, use power-of-2 math for
1358 * switch ports.
1359 */
1360 if (!is_power_of_2(parent_iw))
1361 parent_iw /= 3;
1362 } else {
1363 struct cxl_region_ref *parent_rr;
1364 struct cxl_decoder *parent_cxld;
1365
1366 parent_rr = cxl_rr_load(parent_port, cxlr);
1367 parent_cxld = parent_rr->decoder;
1368 parent_ig = parent_cxld->interleave_granularity;
1369 parent_iw = parent_cxld->interleave_ways;
1370 }
1371
1372 rc = granularity_to_eig(parent_ig, &peig);
1373 if (rc) {
1374 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1375 dev_name(parent_port->uport_dev),
1376 dev_name(&parent_port->dev), parent_ig);
1377 return rc;
1378 }
1379
1380 rc = ways_to_eiw(parent_iw, &peiw);
1381 if (rc) {
1382 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1383 dev_name(parent_port->uport_dev),
1384 dev_name(&parent_port->dev), parent_iw);
1385 return rc;
1386 }
1387
1388 iw = cxl_rr->nr_targets;
1389 rc = ways_to_eiw(iw, &eiw);
1390 if (rc) {
1391 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1392 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1393 return rc;
1394 }
1395
1396 /*
1397 * Interleave granularity is a multiple of @parent_port granularity.
1398 * Multiplier is the parent port interleave ways.
1399 */
1400 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1401 if (rc) {
1402 dev_dbg(&cxlr->dev,
1403 "%s: invalid granularity calculation (%d * %d)\n",
1404 dev_name(&parent_port->dev), parent_ig, parent_iw);
1405 return rc;
1406 }
1407
1408 rc = eig_to_granularity(eig, &ig);
1409 if (rc) {
1410 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1411 dev_name(port->uport_dev), dev_name(&port->dev),
1412 256 << eig);
1413 return rc;
1414 }
1415
1416 if (iw > 8 || iw > cxlsd->nr_targets) {
1417 dev_dbg(&cxlr->dev,
1418 "%s:%s:%s: ways: %d overflows targets: %d\n",
1419 dev_name(port->uport_dev), dev_name(&port->dev),
1420 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1421 return -ENXIO;
1422 }
1423
1424 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1425 if (cxld->interleave_ways != iw ||
1426 cxld->interleave_granularity != ig ||
1427 cxld->hpa_range.start != p->res->start ||
1428 cxld->hpa_range.end != p->res->end ||
1429 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1430 dev_err(&cxlr->dev,
1431 "%s:%s %s expected iw: %d ig: %d %pr\n",
1432 dev_name(port->uport_dev), dev_name(&port->dev),
1433 __func__, iw, ig, p->res);
1434 dev_err(&cxlr->dev,
1435 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1436 dev_name(port->uport_dev), dev_name(&port->dev),
1437 __func__, cxld->interleave_ways,
1438 cxld->interleave_granularity,
1439 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1440 "enabled" :
1441 "disabled",
1442 cxld->hpa_range.start, cxld->hpa_range.end);
1443 return -ENXIO;
1444 }
1445 } else {
1446 rc = check_interleave_cap(cxld, iw, ig);
1447 if (rc) {
1448 dev_dbg(&cxlr->dev,
1449 "%s:%s iw: %d ig: %d is not supported\n",
1450 dev_name(port->uport_dev),
1451 dev_name(&port->dev), iw, ig);
1452 return rc;
1453 }
1454
1455 cxld->interleave_ways = iw;
1456 cxld->interleave_granularity = ig;
1457 cxld->hpa_range = (struct range) {
1458 .start = p->res->start,
1459 .end = p->res->end,
1460 };
1461 }
1462 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1463 dev_name(&port->dev), iw, ig);
1464 add_target:
1465 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1466 dev_dbg(&cxlr->dev,
1467 "%s:%s: targets full trying to add %s:%s at %d\n",
1468 dev_name(port->uport_dev), dev_name(&port->dev),
1469 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1470 return -ENXIO;
1471 }
1472 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1473 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1474 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1475 dev_name(port->uport_dev), dev_name(&port->dev),
1476 dev_name(&cxlsd->cxld.dev),
1477 dev_name(ep->dport->dport_dev),
1478 cxl_rr->nr_targets_set);
1479 return -ENXIO;
1480 }
1481 } else
1482 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1483 inc = 1;
1484 out_target_set:
1485 cxl_rr->nr_targets_set += inc;
1486 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1487 dev_name(port->uport_dev), dev_name(&port->dev),
1488 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1489 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1490
1491 return 0;
1492 }
1493
1494 static void cxl_port_reset_targets(struct cxl_port *port,
1495 struct cxl_region *cxlr)
1496 {
1497 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1498 struct cxl_decoder *cxld;
1499
1500 /*
1501 * After the last endpoint has been detached the entire cxl_rr may now
1502 * be gone.
1503 */
1504 if (!cxl_rr)
1505 return;
1506 cxl_rr->nr_targets_set = 0;
1507
1508 cxld = cxl_rr->decoder;
1509 cxld->hpa_range = (struct range) {
1510 .start = 0,
1511 .end = -1,
1512 };
1513 }
1514
1515 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1516 {
1517 struct cxl_region_params *p = &cxlr->params;
1518 struct cxl_endpoint_decoder *cxled;
1519 struct cxl_dev_state *cxlds;
1520 struct cxl_memdev *cxlmd;
1521 struct cxl_port *iter;
1522 struct cxl_ep *ep;
1523 int i;
1524
1525 /*
1526 * In the auto-discovery case skip automatic teardown since the
1527 * address space is already active
1528 */
1529 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1530 return;
1531
1532 for (i = 0; i < p->nr_targets; i++) {
1533 cxled = p->targets[i];
1534 cxlmd = cxled_to_memdev(cxled);
1535 cxlds = cxlmd->cxlds;
1536
1537 if (cxlds->rcd)
1538 continue;
1539
1540 iter = cxled_to_port(cxled);
1541 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1542 iter = to_cxl_port(iter->dev.parent);
1543
1544 for (ep = cxl_ep_load(iter, cxlmd); iter;
1545 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1546 cxl_port_reset_targets(iter, cxlr);
1547 }
1548 }
1549
1550 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1551 {
1552 struct cxl_region_params *p = &cxlr->params;
1553 struct cxl_endpoint_decoder *cxled;
1554 struct cxl_dev_state *cxlds;
1555 int i, rc, rch = 0, vh = 0;
1556 struct cxl_memdev *cxlmd;
1557 struct cxl_port *iter;
1558 struct cxl_ep *ep;
1559
1560 for (i = 0; i < p->nr_targets; i++) {
1561 cxled = p->targets[i];
1562 cxlmd = cxled_to_memdev(cxled);
1563 cxlds = cxlmd->cxlds;
1564
1565 /* validate that all targets agree on topology */
1566 if (!cxlds->rcd) {
1567 vh++;
1568 } else {
1569 rch++;
1570 continue;
1571 }
1572
1573 iter = cxled_to_port(cxled);
1574 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1575 iter = to_cxl_port(iter->dev.parent);
1576
1577 /*
1578 * Descend the topology tree programming / validating
1579 * targets while looking for conflicts.
1580 */
1581 for (ep = cxl_ep_load(iter, cxlmd); iter;
1582 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1583 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1584 if (rc) {
1585 cxl_region_teardown_targets(cxlr);
1586 return rc;
1587 }
1588 }
1589 }
1590
1591 if (rch && vh) {
1592 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1593 cxl_region_teardown_targets(cxlr);
1594 return -ENXIO;
1595 }
1596
1597 return 0;
1598 }
1599
1600 static int cxl_region_validate_position(struct cxl_region *cxlr,
1601 struct cxl_endpoint_decoder *cxled,
1602 int pos)
1603 {
1604 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1605 struct cxl_region_params *p = &cxlr->params;
1606 int i;
1607
1608 if (pos < 0 || pos >= p->interleave_ways) {
1609 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1610 p->interleave_ways);
1611 return -ENXIO;
1612 }
1613
1614 if (p->targets[pos] == cxled)
1615 return 0;
1616
1617 if (p->targets[pos]) {
1618 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1619 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1620
1621 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1622 pos, dev_name(&cxlmd_target->dev),
1623 dev_name(&cxled_target->cxld.dev));
1624 return -EBUSY;
1625 }
1626
1627 for (i = 0; i < p->interleave_ways; i++) {
1628 struct cxl_endpoint_decoder *cxled_target;
1629 struct cxl_memdev *cxlmd_target;
1630
1631 cxled_target = p->targets[i];
1632 if (!cxled_target)
1633 continue;
1634
1635 cxlmd_target = cxled_to_memdev(cxled_target);
1636 if (cxlmd_target == cxlmd) {
1637 dev_dbg(&cxlr->dev,
1638 "%s already specified at position %d via: %s\n",
1639 dev_name(&cxlmd->dev), pos,
1640 dev_name(&cxled_target->cxld.dev));
1641 return -EBUSY;
1642 }
1643 }
1644
1645 return 0;
1646 }
1647
1648 static int cxl_region_attach_position(struct cxl_region *cxlr,
1649 struct cxl_root_decoder *cxlrd,
1650 struct cxl_endpoint_decoder *cxled,
1651 const struct cxl_dport *dport, int pos)
1652 {
1653 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1654 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1655 struct cxl_decoder *cxld = &cxlsd->cxld;
1656 int iw = cxld->interleave_ways;
1657 struct cxl_port *iter;
1658 int rc;
1659
1660 if (dport != cxlrd->cxlsd.target[pos % iw]) {
1661 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1662 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1663 dev_name(&cxlrd->cxlsd.cxld.dev));
1664 return -ENXIO;
1665 }
1666
1667 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1668 iter = to_cxl_port(iter->dev.parent)) {
1669 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1670 if (rc)
1671 goto err;
1672 }
1673
1674 return 0;
1675
1676 err:
1677 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1678 iter = to_cxl_port(iter->dev.parent))
1679 cxl_port_detach_region(iter, cxlr, cxled);
1680 return rc;
1681 }
1682
1683 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1684 struct cxl_endpoint_decoder *cxled, int pos)
1685 {
1686 struct cxl_region_params *p = &cxlr->params;
1687
1688 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1689 dev_err(&cxlr->dev,
1690 "%s: unable to add decoder to autodetected region\n",
1691 dev_name(&cxled->cxld.dev));
1692 return -EINVAL;
1693 }
1694
1695 if (pos >= 0) {
1696 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1697 dev_name(&cxled->cxld.dev), pos);
1698 return -EINVAL;
1699 }
1700
1701 if (p->nr_targets >= p->interleave_ways) {
1702 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1703 dev_name(&cxled->cxld.dev));
1704 return -ENXIO;
1705 }
1706
1707 /*
1708 * Temporarily record the endpoint decoder into the target array. Yes,
1709 * this means that userspace can view devices in the wrong position
1710 * before the region activates, and must be careful to understand when
1711 * it might be racing region autodiscovery.
1712 */
1713 pos = p->nr_targets;
1714 p->targets[pos] = cxled;
1715 cxled->pos = pos;
1716 p->nr_targets++;
1717
1718 return 0;
1719 }
1720
1721 static int cmp_interleave_pos(const void *a, const void *b)
1722 {
1723 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1724 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1725
1726 return cxled_a->pos - cxled_b->pos;
1727 }
1728
1729 static struct cxl_port *next_port(struct cxl_port *port)
1730 {
1731 if (!port->parent_dport)
1732 return NULL;
1733 return port->parent_dport->port;
1734 }
1735
1736 static int match_switch_decoder_by_range(struct device *dev,
1737 const void *data)
1738 {
1739 struct cxl_switch_decoder *cxlsd;
1740 const struct range *r1, *r2 = data;
1741
1742
1743 if (!is_switch_decoder(dev))
1744 return 0;
1745
1746 cxlsd = to_cxl_switch_decoder(dev);
1747 r1 = &cxlsd->cxld.hpa_range;
1748
1749 if (is_root_decoder(dev))
1750 return range_contains(r1, r2);
1751 return (r1->start == r2->start && r1->end == r2->end);
1752 }
1753
1754 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1755 int *pos, int *ways)
1756 {
1757 struct cxl_switch_decoder *cxlsd;
1758 struct cxl_port *parent;
1759 struct device *dev;
1760 int rc = -ENXIO;
1761
1762 parent = next_port(port);
1763 if (!parent)
1764 return rc;
1765
1766 dev = device_find_child(&parent->dev, range,
1767 match_switch_decoder_by_range);
1768 if (!dev) {
1769 dev_err(port->uport_dev,
1770 "failed to find decoder mapping %#llx-%#llx\n",
1771 range->start, range->end);
1772 return rc;
1773 }
1774 cxlsd = to_cxl_switch_decoder(dev);
1775 *ways = cxlsd->cxld.interleave_ways;
1776
1777 for (int i = 0; i < *ways; i++) {
1778 if (cxlsd->target[i] == port->parent_dport) {
1779 *pos = i;
1780 rc = 0;
1781 break;
1782 }
1783 }
1784 put_device(dev);
1785
1786 return rc;
1787 }
1788
1789 /**
1790 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1791 * @cxled: endpoint decoder member of given region
1792 *
1793 * The endpoint position is calculated by traversing the topology from
1794 * the endpoint to the root decoder and iteratively applying this
1795 * calculation:
1796 *
1797 * position = position * parent_ways + parent_pos;
1798 *
1799 * ...where @position is inferred from switch and root decoder target lists.
1800 *
1801 * Return: position >= 0 on success
1802 * -ENXIO on failure
1803 */
1804 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1805 {
1806 struct cxl_port *iter, *port = cxled_to_port(cxled);
1807 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1808 struct range *range = &cxled->cxld.hpa_range;
1809 int parent_ways = 0, parent_pos = 0, pos = 0;
1810 int rc;
1811
1812 /*
1813 * Example: the expected interleave order of the 4-way region shown
1814 * below is: mem0, mem2, mem1, mem3
1815 *
1816 * root_port
1817 * / \
1818 * host_bridge_0 host_bridge_1
1819 * | | | |
1820 * mem0 mem1 mem2 mem3
1821 *
1822 * In the example the calculator will iterate twice. The first iteration
1823 * uses the mem position in the host-bridge and the ways of the host-
1824 * bridge to generate the first, or local, position. The second
1825 * iteration uses the host-bridge position in the root_port and the ways
1826 * of the root_port to refine the position.
1827 *
1828 * A trace of the calculation per endpoint looks like this:
1829 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1830 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1831 * pos: 0 pos: 1
1832 *
1833 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1834 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1835 * pos: 2 pos = 3
1836 *
1837 * Note that while this example is simple, the method applies to more
1838 * complex topologies, including those with switches.
1839 */
1840
1841 /* Iterate from endpoint to root_port refining the position */
1842 for (iter = port; iter; iter = next_port(iter)) {
1843 if (is_cxl_root(iter))
1844 break;
1845
1846 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1847 if (rc)
1848 return rc;
1849
1850 pos = pos * parent_ways + parent_pos;
1851 }
1852
1853 dev_dbg(&cxlmd->dev,
1854 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1855 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1856 dev_name(&port->dev), range->start, range->end, pos);
1857
1858 return pos;
1859 }
1860
1861 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1862 {
1863 struct cxl_region_params *p = &cxlr->params;
1864 int i, rc = 0;
1865
1866 for (i = 0; i < p->nr_targets; i++) {
1867 struct cxl_endpoint_decoder *cxled = p->targets[i];
1868
1869 cxled->pos = cxl_calc_interleave_pos(cxled);
1870 /*
1871 * Record that sorting failed, but still continue to calc
1872 * cxled->pos so that follow-on code paths can reliably
1873 * do p->targets[cxled->pos] to self-reference their entry.
1874 */
1875 if (cxled->pos < 0)
1876 rc = -ENXIO;
1877 }
1878 /* Keep the cxlr target list in interleave position order */
1879 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1880 cmp_interleave_pos, NULL);
1881
1882 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1883 return rc;
1884 }
1885
1886 static int cxl_region_attach(struct cxl_region *cxlr,
1887 struct cxl_endpoint_decoder *cxled, int pos)
1888 {
1889 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1890 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1891 struct cxl_region_params *p = &cxlr->params;
1892 struct cxl_port *ep_port, *root_port;
1893 struct cxl_dport *dport;
1894 int rc = -ENXIO;
1895
1896 rc = check_interleave_cap(&cxled->cxld, p->interleave_ways,
1897 p->interleave_granularity);
1898 if (rc) {
1899 dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
1900 dev_name(&cxled->cxld.dev), p->interleave_ways,
1901 p->interleave_granularity);
1902 return rc;
1903 }
1904
1905 if (cxled->mode != cxlr->mode) {
1906 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1907 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1908 return -EINVAL;
1909 }
1910
1911 if (cxled->mode == CXL_DECODER_DEAD) {
1912 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1913 return -ENODEV;
1914 }
1915
1916 /* all full of members, or interleave config not established? */
1917 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1918 dev_dbg(&cxlr->dev, "region already active\n");
1919 return -EBUSY;
1920 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1921 dev_dbg(&cxlr->dev, "interleave config missing\n");
1922 return -ENXIO;
1923 }
1924
1925 if (p->nr_targets >= p->interleave_ways) {
1926 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1927 p->nr_targets);
1928 return -EINVAL;
1929 }
1930
1931 ep_port = cxled_to_port(cxled);
1932 root_port = cxlrd_to_port(cxlrd);
1933 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1934 if (!dport) {
1935 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1936 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1937 dev_name(cxlr->dev.parent));
1938 return -ENXIO;
1939 }
1940
1941 if (cxled->cxld.target_type != cxlr->type) {
1942 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1943 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1944 cxled->cxld.target_type, cxlr->type);
1945 return -ENXIO;
1946 }
1947
1948 if (!cxled->dpa_res) {
1949 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1950 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1951 return -ENXIO;
1952 }
1953
1954 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1955 resource_size(p->res)) {
1956 dev_dbg(&cxlr->dev,
1957 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1958 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1959 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1960 (u64)resource_size(p->res));
1961 return -EINVAL;
1962 }
1963
1964 cxl_region_perf_data_calculate(cxlr, cxled);
1965
1966 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1967 int i;
1968
1969 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1970 if (rc)
1971 return rc;
1972
1973 /* await more targets to arrive... */
1974 if (p->nr_targets < p->interleave_ways)
1975 return 0;
1976
1977 /*
1978 * All targets are here, which implies all PCI enumeration that
1979 * affects this region has been completed. Walk the topology to
1980 * sort the devices into their relative region decode position.
1981 */
1982 rc = cxl_region_sort_targets(cxlr);
1983 if (rc)
1984 return rc;
1985
1986 for (i = 0; i < p->nr_targets; i++) {
1987 cxled = p->targets[i];
1988 ep_port = cxled_to_port(cxled);
1989 dport = cxl_find_dport_by_dev(root_port,
1990 ep_port->host_bridge);
1991 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1992 dport, i);
1993 if (rc)
1994 return rc;
1995 }
1996
1997 rc = cxl_region_setup_targets(cxlr);
1998 if (rc)
1999 return rc;
2000
2001 /*
2002 * If target setup succeeds in the autodiscovery case
2003 * then the region is already committed.
2004 */
2005 p->state = CXL_CONFIG_COMMIT;
2006 cxl_region_shared_upstream_bandwidth_update(cxlr);
2007
2008 return 0;
2009 }
2010
2011 rc = cxl_region_validate_position(cxlr, cxled, pos);
2012 if (rc)
2013 return rc;
2014
2015 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
2016 if (rc)
2017 return rc;
2018
2019 p->targets[pos] = cxled;
2020 cxled->pos = pos;
2021 p->nr_targets++;
2022
2023 if (p->nr_targets == p->interleave_ways) {
2024 rc = cxl_region_setup_targets(cxlr);
2025 if (rc)
2026 return rc;
2027 p->state = CXL_CONFIG_ACTIVE;
2028 cxl_region_shared_upstream_bandwidth_update(cxlr);
2029 }
2030
2031 cxled->cxld.interleave_ways = p->interleave_ways;
2032 cxled->cxld.interleave_granularity = p->interleave_granularity;
2033 cxled->cxld.hpa_range = (struct range) {
2034 .start = p->res->start,
2035 .end = p->res->end,
2036 };
2037
2038 if (p->nr_targets != p->interleave_ways)
2039 return 0;
2040
2041 /*
2042 * Test the auto-discovery position calculator function
2043 * against this successfully created user-defined region.
2044 * A fail message here means that this interleave config
2045 * will fail when presented as CXL_REGION_F_AUTO.
2046 */
2047 for (int i = 0; i < p->nr_targets; i++) {
2048 struct cxl_endpoint_decoder *cxled = p->targets[i];
2049 int test_pos;
2050
2051 test_pos = cxl_calc_interleave_pos(cxled);
2052 dev_dbg(&cxled->cxld.dev,
2053 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
2054 (test_pos == cxled->pos) ? "success" : "fail",
2055 test_pos, cxled->pos);
2056 }
2057
2058 return 0;
2059 }
2060
2061 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
2062 {
2063 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
2064 struct cxl_region *cxlr = cxled->cxld.region;
2065 struct cxl_region_params *p;
2066 int rc = 0;
2067
2068 lockdep_assert_held_write(&cxl_region_rwsem);
2069
2070 if (!cxlr)
2071 return 0;
2072
2073 p = &cxlr->params;
2074 get_device(&cxlr->dev);
2075
2076 if (p->state > CXL_CONFIG_ACTIVE) {
2077 cxl_region_decode_reset(cxlr, p->interleave_ways);
2078 p->state = CXL_CONFIG_ACTIVE;
2079 }
2080
2081 for (iter = ep_port; !is_cxl_root(iter);
2082 iter = to_cxl_port(iter->dev.parent))
2083 cxl_port_detach_region(iter, cxlr, cxled);
2084
2085 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
2086 p->targets[cxled->pos] != cxled) {
2087 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2088
2089 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
2090 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2091 cxled->pos);
2092 goto out;
2093 }
2094
2095 if (p->state == CXL_CONFIG_ACTIVE) {
2096 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2097 cxl_region_teardown_targets(cxlr);
2098 }
2099 p->targets[cxled->pos] = NULL;
2100 p->nr_targets--;
2101 cxled->cxld.hpa_range = (struct range) {
2102 .start = 0,
2103 .end = -1,
2104 };
2105
2106 /* notify the region driver that one of its targets has departed */
2107 up_write(&cxl_region_rwsem);
2108 device_release_driver(&cxlr->dev);
2109 down_write(&cxl_region_rwsem);
2110 out:
2111 put_device(&cxlr->dev);
2112 return rc;
2113 }
2114
2115 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
2116 {
2117 down_write(&cxl_region_rwsem);
2118 cxled->mode = CXL_DECODER_DEAD;
2119 cxl_region_detach(cxled);
2120 up_write(&cxl_region_rwsem);
2121 }
2122
2123 static int attach_target(struct cxl_region *cxlr,
2124 struct cxl_endpoint_decoder *cxled, int pos,
2125 unsigned int state)
2126 {
2127 int rc = 0;
2128
2129 if (state == TASK_INTERRUPTIBLE)
2130 rc = down_write_killable(&cxl_region_rwsem);
2131 else
2132 down_write(&cxl_region_rwsem);
2133 if (rc)
2134 return rc;
2135
2136 down_read(&cxl_dpa_rwsem);
2137 rc = cxl_region_attach(cxlr, cxled, pos);
2138 up_read(&cxl_dpa_rwsem);
2139 up_write(&cxl_region_rwsem);
2140 return rc;
2141 }
2142
2143 static int detach_target(struct cxl_region *cxlr, int pos)
2144 {
2145 struct cxl_region_params *p = &cxlr->params;
2146 int rc;
2147
2148 rc = down_write_killable(&cxl_region_rwsem);
2149 if (rc)
2150 return rc;
2151
2152 if (pos >= p->interleave_ways) {
2153 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
2154 p->interleave_ways);
2155 rc = -ENXIO;
2156 goto out;
2157 }
2158
2159 if (!p->targets[pos]) {
2160 rc = 0;
2161 goto out;
2162 }
2163
2164 rc = cxl_region_detach(p->targets[pos]);
2165 out:
2166 up_write(&cxl_region_rwsem);
2167 return rc;
2168 }
2169
2170 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2171 size_t len)
2172 {
2173 int rc;
2174
2175 if (sysfs_streq(buf, "\n"))
2176 rc = detach_target(cxlr, pos);
2177 else {
2178 struct device *dev;
2179
2180 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2181 if (!dev)
2182 return -ENODEV;
2183
2184 if (!is_endpoint_decoder(dev)) {
2185 rc = -EINVAL;
2186 goto out;
2187 }
2188
2189 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2190 TASK_INTERRUPTIBLE);
2191 out:
2192 put_device(dev);
2193 }
2194
2195 if (rc < 0)
2196 return rc;
2197 return len;
2198 }
2199
2200 #define TARGET_ATTR_RW(n) \
2201 static ssize_t target##n##_show( \
2202 struct device *dev, struct device_attribute *attr, char *buf) \
2203 { \
2204 return show_targetN(to_cxl_region(dev), buf, (n)); \
2205 } \
2206 static ssize_t target##n##_store(struct device *dev, \
2207 struct device_attribute *attr, \
2208 const char *buf, size_t len) \
2209 { \
2210 return store_targetN(to_cxl_region(dev), buf, (n), len); \
2211 } \
2212 static DEVICE_ATTR_RW(target##n)
2213
2214 TARGET_ATTR_RW(0);
2215 TARGET_ATTR_RW(1);
2216 TARGET_ATTR_RW(2);
2217 TARGET_ATTR_RW(3);
2218 TARGET_ATTR_RW(4);
2219 TARGET_ATTR_RW(5);
2220 TARGET_ATTR_RW(6);
2221 TARGET_ATTR_RW(7);
2222 TARGET_ATTR_RW(8);
2223 TARGET_ATTR_RW(9);
2224 TARGET_ATTR_RW(10);
2225 TARGET_ATTR_RW(11);
2226 TARGET_ATTR_RW(12);
2227 TARGET_ATTR_RW(13);
2228 TARGET_ATTR_RW(14);
2229 TARGET_ATTR_RW(15);
2230
2231 static struct attribute *target_attrs[] = {
2232 &dev_attr_target0.attr,
2233 &dev_attr_target1.attr,
2234 &dev_attr_target2.attr,
2235 &dev_attr_target3.attr,
2236 &dev_attr_target4.attr,
2237 &dev_attr_target5.attr,
2238 &dev_attr_target6.attr,
2239 &dev_attr_target7.attr,
2240 &dev_attr_target8.attr,
2241 &dev_attr_target9.attr,
2242 &dev_attr_target10.attr,
2243 &dev_attr_target11.attr,
2244 &dev_attr_target12.attr,
2245 &dev_attr_target13.attr,
2246 &dev_attr_target14.attr,
2247 &dev_attr_target15.attr,
2248 NULL,
2249 };
2250
2251 static umode_t cxl_region_target_visible(struct kobject *kobj,
2252 struct attribute *a, int n)
2253 {
2254 struct device *dev = kobj_to_dev(kobj);
2255 struct cxl_region *cxlr = to_cxl_region(dev);
2256 struct cxl_region_params *p = &cxlr->params;
2257
2258 if (n < p->interleave_ways)
2259 return a->mode;
2260 return 0;
2261 }
2262
2263 static const struct attribute_group cxl_region_target_group = {
2264 .attrs = target_attrs,
2265 .is_visible = cxl_region_target_visible,
2266 };
2267
2268 static const struct attribute_group *get_cxl_region_target_group(void)
2269 {
2270 return &cxl_region_target_group;
2271 }
2272
2273 static const struct attribute_group *region_groups[] = {
2274 &cxl_base_attribute_group,
2275 &cxl_region_group,
2276 &cxl_region_target_group,
2277 &cxl_region_access0_coordinate_group,
2278 &cxl_region_access1_coordinate_group,
2279 NULL,
2280 };
2281
2282 static void cxl_region_release(struct device *dev)
2283 {
2284 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2285 struct cxl_region *cxlr = to_cxl_region(dev);
2286 int id = atomic_read(&cxlrd->region_id);
2287
2288 /*
2289 * Try to reuse the recently idled id rather than the cached
2290 * next id to prevent the region id space from increasing
2291 * unnecessarily.
2292 */
2293 if (cxlr->id < id)
2294 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2295 memregion_free(id);
2296 goto out;
2297 }
2298
2299 memregion_free(cxlr->id);
2300 out:
2301 put_device(dev->parent);
2302 kfree(cxlr);
2303 }
2304
2305 const struct device_type cxl_region_type = {
2306 .name = "cxl_region",
2307 .release = cxl_region_release,
2308 .groups = region_groups
2309 };
2310
2311 bool is_cxl_region(struct device *dev)
2312 {
2313 return dev->type == &cxl_region_type;
2314 }
2315 EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL");
2316
2317 static struct cxl_region *to_cxl_region(struct device *dev)
2318 {
2319 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2320 "not a cxl_region device\n"))
2321 return NULL;
2322
2323 return container_of(dev, struct cxl_region, dev);
2324 }
2325
2326 static void unregister_region(void *_cxlr)
2327 {
2328 struct cxl_region *cxlr = _cxlr;
2329 struct cxl_region_params *p = &cxlr->params;
2330 int i;
2331
2332 device_del(&cxlr->dev);
2333
2334 /*
2335 * Now that region sysfs is shutdown, the parameter block is now
2336 * read-only, so no need to hold the region rwsem to access the
2337 * region parameters.
2338 */
2339 for (i = 0; i < p->interleave_ways; i++)
2340 detach_target(cxlr, i);
2341
2342 cxl_region_iomem_release(cxlr);
2343 put_device(&cxlr->dev);
2344 }
2345
2346 static struct lock_class_key cxl_region_key;
2347
2348 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2349 {
2350 struct cxl_region *cxlr;
2351 struct device *dev;
2352
2353 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2354 if (!cxlr) {
2355 memregion_free(id);
2356 return ERR_PTR(-ENOMEM);
2357 }
2358
2359 dev = &cxlr->dev;
2360 device_initialize(dev);
2361 lockdep_set_class(&dev->mutex, &cxl_region_key);
2362 dev->parent = &cxlrd->cxlsd.cxld.dev;
2363 /*
2364 * Keep root decoder pinned through cxl_region_release to fixup
2365 * region id allocations
2366 */
2367 get_device(dev->parent);
2368 device_set_pm_not_required(dev);
2369 dev->bus = &cxl_bus_type;
2370 dev->type = &cxl_region_type;
2371 cxlr->id = id;
2372
2373 return cxlr;
2374 }
2375
2376 static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2377 {
2378 int cset = 0;
2379 int rc;
2380
2381 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2382 if (cxlr->coord[i].read_bandwidth) {
2383 rc = 0;
2384 if (cxl_need_node_perf_attrs_update(nid))
2385 node_set_perf_attrs(nid, &cxlr->coord[i], i);
2386 else
2387 rc = cxl_update_hmat_access_coordinates(nid, cxlr, i);
2388
2389 if (rc == 0)
2390 cset++;
2391 }
2392 }
2393
2394 if (!cset)
2395 return false;
2396
2397 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group());
2398 if (rc)
2399 dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2400
2401 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group());
2402 if (rc)
2403 dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2404
2405 return true;
2406 }
2407
2408 static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2409 unsigned long action, void *arg)
2410 {
2411 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2412 memory_notifier);
2413 struct memory_notify *mnb = arg;
2414 int nid = mnb->status_change_nid;
2415 int region_nid;
2416
2417 if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
2418 return NOTIFY_DONE;
2419
2420 /*
2421 * No need to hold cxl_region_rwsem; region parameters are stable
2422 * within the cxl_region driver.
2423 */
2424 region_nid = phys_to_target_node(cxlr->params.res->start);
2425 if (nid != region_nid)
2426 return NOTIFY_DONE;
2427
2428 if (!cxl_region_update_coordinates(cxlr, nid))
2429 return NOTIFY_DONE;
2430
2431 return NOTIFY_OK;
2432 }
2433
2434 static int cxl_region_calculate_adistance(struct notifier_block *nb,
2435 unsigned long nid, void *data)
2436 {
2437 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2438 adist_notifier);
2439 struct access_coordinate *perf;
2440 int *adist = data;
2441 int region_nid;
2442
2443 /*
2444 * No need to hold cxl_region_rwsem; region parameters are stable
2445 * within the cxl_region driver.
2446 */
2447 region_nid = phys_to_target_node(cxlr->params.res->start);
2448 if (nid != region_nid)
2449 return NOTIFY_OK;
2450
2451 perf = &cxlr->coord[ACCESS_COORDINATE_CPU];
2452
2453 if (mt_perf_to_adistance(perf, adist))
2454 return NOTIFY_OK;
2455
2456 return NOTIFY_STOP;
2457 }
2458
2459 /**
2460 * devm_cxl_add_region - Adds a region to a decoder
2461 * @cxlrd: root decoder
2462 * @id: memregion id to create, or memregion_free() on failure
2463 * @mode: mode for the endpoint decoders of this region
2464 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2465 *
2466 * This is the second step of region initialization. Regions exist within an
2467 * address space which is mapped by a @cxlrd.
2468 *
2469 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2470 * code. The region will be named "regionZ" where Z is the unique region number.
2471 */
2472 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2473 int id,
2474 enum cxl_decoder_mode mode,
2475 enum cxl_decoder_type type)
2476 {
2477 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2478 struct cxl_region *cxlr;
2479 struct device *dev;
2480 int rc;
2481
2482 cxlr = cxl_region_alloc(cxlrd, id);
2483 if (IS_ERR(cxlr))
2484 return cxlr;
2485 cxlr->mode = mode;
2486 cxlr->type = type;
2487
2488 dev = &cxlr->dev;
2489 rc = dev_set_name(dev, "region%d", id);
2490 if (rc)
2491 goto err;
2492
2493 rc = device_add(dev);
2494 if (rc)
2495 goto err;
2496
2497 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2498 if (rc)
2499 return ERR_PTR(rc);
2500
2501 dev_dbg(port->uport_dev, "%s: created %s\n",
2502 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2503 return cxlr;
2504
2505 err:
2506 put_device(dev);
2507 return ERR_PTR(rc);
2508 }
2509
2510 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2511 {
2512 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2513 }
2514
2515 static ssize_t create_pmem_region_show(struct device *dev,
2516 struct device_attribute *attr, char *buf)
2517 {
2518 return __create_region_show(to_cxl_root_decoder(dev), buf);
2519 }
2520
2521 static ssize_t create_ram_region_show(struct device *dev,
2522 struct device_attribute *attr, char *buf)
2523 {
2524 return __create_region_show(to_cxl_root_decoder(dev), buf);
2525 }
2526
2527 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2528 enum cxl_decoder_mode mode, int id)
2529 {
2530 int rc;
2531
2532 switch (mode) {
2533 case CXL_DECODER_RAM:
2534 case CXL_DECODER_PMEM:
2535 break;
2536 default:
2537 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2538 return ERR_PTR(-EINVAL);
2539 }
2540
2541 rc = memregion_alloc(GFP_KERNEL);
2542 if (rc < 0)
2543 return ERR_PTR(rc);
2544
2545 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2546 memregion_free(rc);
2547 return ERR_PTR(-EBUSY);
2548 }
2549
2550 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2551 }
2552
2553 static ssize_t create_region_store(struct device *dev, const char *buf,
2554 size_t len, enum cxl_decoder_mode mode)
2555 {
2556 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2557 struct cxl_region *cxlr;
2558 int rc, id;
2559
2560 rc = sscanf(buf, "region%d\n", &id);
2561 if (rc != 1)
2562 return -EINVAL;
2563
2564 cxlr = __create_region(cxlrd, mode, id);
2565 if (IS_ERR(cxlr))
2566 return PTR_ERR(cxlr);
2567
2568 return len;
2569 }
2570
2571 static ssize_t create_pmem_region_store(struct device *dev,
2572 struct device_attribute *attr,
2573 const char *buf, size_t len)
2574 {
2575 return create_region_store(dev, buf, len, CXL_DECODER_PMEM);
2576 }
2577 DEVICE_ATTR_RW(create_pmem_region);
2578
2579 static ssize_t create_ram_region_store(struct device *dev,
2580 struct device_attribute *attr,
2581 const char *buf, size_t len)
2582 {
2583 return create_region_store(dev, buf, len, CXL_DECODER_RAM);
2584 }
2585 DEVICE_ATTR_RW(create_ram_region);
2586
2587 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2588 char *buf)
2589 {
2590 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2591 ssize_t rc;
2592
2593 rc = down_read_interruptible(&cxl_region_rwsem);
2594 if (rc)
2595 return rc;
2596
2597 if (cxld->region)
2598 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2599 else
2600 rc = sysfs_emit(buf, "\n");
2601 up_read(&cxl_region_rwsem);
2602
2603 return rc;
2604 }
2605 DEVICE_ATTR_RO(region);
2606
2607 static struct cxl_region *
2608 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2609 {
2610 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2611 struct device *region_dev;
2612
2613 region_dev = device_find_child_by_name(&cxld->dev, name);
2614 if (!region_dev)
2615 return ERR_PTR(-ENODEV);
2616
2617 return to_cxl_region(region_dev);
2618 }
2619
2620 static ssize_t delete_region_store(struct device *dev,
2621 struct device_attribute *attr,
2622 const char *buf, size_t len)
2623 {
2624 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2625 struct cxl_port *port = to_cxl_port(dev->parent);
2626 struct cxl_region *cxlr;
2627
2628 cxlr = cxl_find_region_by_name(cxlrd, buf);
2629 if (IS_ERR(cxlr))
2630 return PTR_ERR(cxlr);
2631
2632 devm_release_action(port->uport_dev, unregister_region, cxlr);
2633 put_device(&cxlr->dev);
2634
2635 return len;
2636 }
2637 DEVICE_ATTR_WO(delete_region);
2638
2639 static void cxl_pmem_region_release(struct device *dev)
2640 {
2641 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2642 int i;
2643
2644 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2645 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2646
2647 put_device(&cxlmd->dev);
2648 }
2649
2650 kfree(cxlr_pmem);
2651 }
2652
2653 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2654 &cxl_base_attribute_group,
2655 NULL,
2656 };
2657
2658 const struct device_type cxl_pmem_region_type = {
2659 .name = "cxl_pmem_region",
2660 .release = cxl_pmem_region_release,
2661 .groups = cxl_pmem_region_attribute_groups,
2662 };
2663
2664 bool is_cxl_pmem_region(struct device *dev)
2665 {
2666 return dev->type == &cxl_pmem_region_type;
2667 }
2668 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL");
2669
2670 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2671 {
2672 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2673 "not a cxl_pmem_region device\n"))
2674 return NULL;
2675 return container_of(dev, struct cxl_pmem_region, dev);
2676 }
2677 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL");
2678
2679 struct cxl_poison_context {
2680 struct cxl_port *port;
2681 enum cxl_decoder_mode mode;
2682 u64 offset;
2683 };
2684
2685 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2686 struct cxl_poison_context *ctx)
2687 {
2688 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2689 u64 offset, length;
2690 int rc = 0;
2691
2692 /*
2693 * Collect poison for the remaining unmapped resources
2694 * after poison is collected by committed endpoints.
2695 *
2696 * Knowing that PMEM must always follow RAM, get poison
2697 * for unmapped resources based on the last decoder's mode:
2698 * ram: scan remains of ram range, then any pmem range
2699 * pmem: scan remains of pmem range
2700 */
2701
2702 if (ctx->mode == CXL_DECODER_RAM) {
2703 offset = ctx->offset;
2704 length = resource_size(&cxlds->ram_res) - offset;
2705 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2706 if (rc == -EFAULT)
2707 rc = 0;
2708 if (rc)
2709 return rc;
2710 }
2711 if (ctx->mode == CXL_DECODER_PMEM) {
2712 offset = ctx->offset;
2713 length = resource_size(&cxlds->dpa_res) - offset;
2714 if (!length)
2715 return 0;
2716 } else if (resource_size(&cxlds->pmem_res)) {
2717 offset = cxlds->pmem_res.start;
2718 length = resource_size(&cxlds->pmem_res);
2719 } else {
2720 return 0;
2721 }
2722
2723 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2724 }
2725
2726 static int poison_by_decoder(struct device *dev, void *arg)
2727 {
2728 struct cxl_poison_context *ctx = arg;
2729 struct cxl_endpoint_decoder *cxled;
2730 struct cxl_memdev *cxlmd;
2731 u64 offset, length;
2732 int rc = 0;
2733
2734 if (!is_endpoint_decoder(dev))
2735 return rc;
2736
2737 cxled = to_cxl_endpoint_decoder(dev);
2738 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2739 return rc;
2740
2741 /*
2742 * Regions are only created with single mode decoders: pmem or ram.
2743 * Linux does not support mixed mode decoders. This means that
2744 * reading poison per endpoint decoder adheres to the requirement
2745 * that poison reads of pmem and ram must be separated.
2746 * CXL 3.0 Spec 8.2.9.8.4.1
2747 */
2748 if (cxled->mode == CXL_DECODER_MIXED) {
2749 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2750 return rc;
2751 }
2752
2753 cxlmd = cxled_to_memdev(cxled);
2754 if (cxled->skip) {
2755 offset = cxled->dpa_res->start - cxled->skip;
2756 length = cxled->skip;
2757 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2758 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2759 rc = 0;
2760 if (rc)
2761 return rc;
2762 }
2763
2764 offset = cxled->dpa_res->start;
2765 length = cxled->dpa_res->end - offset + 1;
2766 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2767 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2768 rc = 0;
2769 if (rc)
2770 return rc;
2771
2772 /* Iterate until commit_end is reached */
2773 if (cxled->cxld.id == ctx->port->commit_end) {
2774 ctx->offset = cxled->dpa_res->end + 1;
2775 ctx->mode = cxled->mode;
2776 return 1;
2777 }
2778
2779 return 0;
2780 }
2781
2782 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2783 {
2784 struct cxl_poison_context ctx;
2785 int rc = 0;
2786
2787 ctx = (struct cxl_poison_context) {
2788 .port = port
2789 };
2790
2791 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2792 if (rc == 1)
2793 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2794 &ctx);
2795
2796 return rc;
2797 }
2798
2799 struct cxl_dpa_to_region_context {
2800 struct cxl_region *cxlr;
2801 u64 dpa;
2802 };
2803
2804 static int __cxl_dpa_to_region(struct device *dev, void *arg)
2805 {
2806 struct cxl_dpa_to_region_context *ctx = arg;
2807 struct cxl_endpoint_decoder *cxled;
2808 struct cxl_region *cxlr;
2809 u64 dpa = ctx->dpa;
2810
2811 if (!is_endpoint_decoder(dev))
2812 return 0;
2813
2814 cxled = to_cxl_endpoint_decoder(dev);
2815 if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res))
2816 return 0;
2817
2818 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
2819 return 0;
2820
2821 /*
2822 * Stop the region search (return 1) when an endpoint mapping is
2823 * found. The region may not be fully constructed so offering
2824 * the cxlr in the context structure is not guaranteed.
2825 */
2826 cxlr = cxled->cxld.region;
2827 if (cxlr)
2828 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2829 dev_name(&cxlr->dev));
2830 else
2831 dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa,
2832 dev_name(dev));
2833
2834 ctx->cxlr = cxlr;
2835
2836 return 1;
2837 }
2838
2839 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2840 {
2841 struct cxl_dpa_to_region_context ctx;
2842 struct cxl_port *port;
2843
2844 ctx = (struct cxl_dpa_to_region_context) {
2845 .dpa = dpa,
2846 };
2847 port = cxlmd->endpoint;
2848 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2849 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
2850
2851 return ctx.cxlr;
2852 }
2853
2854 static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos)
2855 {
2856 struct cxl_region_params *p = &cxlr->params;
2857 int gran = p->interleave_granularity;
2858 int ways = p->interleave_ways;
2859 u64 offset;
2860
2861 /* Is the hpa in an expected chunk for its pos(-ition) */
2862 offset = hpa - p->res->start;
2863 offset = do_div(offset, gran * ways);
2864 if ((offset >= pos * gran) && (offset < (pos + 1) * gran))
2865 return true;
2866
2867 dev_dbg(&cxlr->dev,
2868 "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa);
2869
2870 return false;
2871 }
2872
2873 u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
2874 u64 dpa)
2875 {
2876 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
2877 u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa;
2878 struct cxl_region_params *p = &cxlr->params;
2879 struct cxl_endpoint_decoder *cxled = NULL;
2880 u16 eig = 0;
2881 u8 eiw = 0;
2882 int pos;
2883
2884 for (int i = 0; i < p->nr_targets; i++) {
2885 cxled = p->targets[i];
2886 if (cxlmd == cxled_to_memdev(cxled))
2887 break;
2888 }
2889 if (!cxled || cxlmd != cxled_to_memdev(cxled))
2890 return ULLONG_MAX;
2891
2892 pos = cxled->pos;
2893 ways_to_eiw(p->interleave_ways, &eiw);
2894 granularity_to_eig(p->interleave_granularity, &eig);
2895
2896 /*
2897 * The device position in the region interleave set was removed
2898 * from the offset at HPA->DPA translation. To reconstruct the
2899 * HPA, place the 'pos' in the offset.
2900 *
2901 * The placement of 'pos' in the HPA is determined by interleave
2902 * ways and granularity and is defined in the CXL Spec 3.0 Section
2903 * 8.2.4.19.13 Implementation Note: Device Decode Logic
2904 */
2905
2906 /* Remove the dpa base */
2907 dpa_offset = dpa - cxl_dpa_resource_start(cxled);
2908
2909 mask_upper = GENMASK_ULL(51, eig + 8);
2910
2911 if (eiw < 8) {
2912 hpa_offset = (dpa_offset & mask_upper) << eiw;
2913 hpa_offset |= pos << (eig + 8);
2914 } else {
2915 bits_upper = (dpa_offset & mask_upper) >> (eig + 8);
2916 bits_upper = bits_upper * 3;
2917 hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8);
2918 }
2919
2920 /* The lower bits remain unchanged */
2921 hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0);
2922
2923 /* Apply the hpa_offset to the region base address */
2924 hpa = hpa_offset + p->res->start;
2925
2926 /* Root decoder translation overrides typical modulo decode */
2927 if (cxlrd->hpa_to_spa)
2928 hpa = cxlrd->hpa_to_spa(cxlrd, hpa);
2929
2930 if (hpa < p->res->start || hpa > p->res->end) {
2931 dev_dbg(&cxlr->dev,
2932 "Addr trans fail: hpa 0x%llx not in region\n", hpa);
2933 return ULLONG_MAX;
2934 }
2935
2936 /* Simple chunk check, by pos & gran, only applies to modulo decodes */
2937 if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos)))
2938 return ULLONG_MAX;
2939
2940 return hpa;
2941 }
2942
2943 static struct lock_class_key cxl_pmem_region_key;
2944
2945 static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
2946 {
2947 struct cxl_region_params *p = &cxlr->params;
2948 struct cxl_nvdimm_bridge *cxl_nvb;
2949 struct device *dev;
2950 int i;
2951
2952 guard(rwsem_read)(&cxl_region_rwsem);
2953 if (p->state != CXL_CONFIG_COMMIT)
2954 return -ENXIO;
2955
2956 struct cxl_pmem_region *cxlr_pmem __free(kfree) =
2957 kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL);
2958 if (!cxlr_pmem)
2959 return -ENOMEM;
2960
2961 cxlr_pmem->hpa_range.start = p->res->start;
2962 cxlr_pmem->hpa_range.end = p->res->end;
2963
2964 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2965 cxlr_pmem->nr_mappings = p->nr_targets;
2966 for (i = 0; i < p->nr_targets; i++) {
2967 struct cxl_endpoint_decoder *cxled = p->targets[i];
2968 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2969 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2970
2971 /*
2972 * Regions never span CXL root devices, so by definition the
2973 * bridge for one device is the same for all.
2974 */
2975 if (i == 0) {
2976 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint);
2977 if (!cxl_nvb)
2978 return -ENODEV;
2979 cxlr->cxl_nvb = cxl_nvb;
2980 }
2981 m->cxlmd = cxlmd;
2982 get_device(&cxlmd->dev);
2983 m->start = cxled->dpa_res->start;
2984 m->size = resource_size(cxled->dpa_res);
2985 m->position = i;
2986 }
2987
2988 dev = &cxlr_pmem->dev;
2989 device_initialize(dev);
2990 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2991 device_set_pm_not_required(dev);
2992 dev->parent = &cxlr->dev;
2993 dev->bus = &cxl_bus_type;
2994 dev->type = &cxl_pmem_region_type;
2995 cxlr_pmem->cxlr = cxlr;
2996 cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem);
2997
2998 return 0;
2999 }
3000
3001 static void cxl_dax_region_release(struct device *dev)
3002 {
3003 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
3004
3005 kfree(cxlr_dax);
3006 }
3007
3008 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
3009 &cxl_base_attribute_group,
3010 NULL,
3011 };
3012
3013 const struct device_type cxl_dax_region_type = {
3014 .name = "cxl_dax_region",
3015 .release = cxl_dax_region_release,
3016 .groups = cxl_dax_region_attribute_groups,
3017 };
3018
3019 static bool is_cxl_dax_region(struct device *dev)
3020 {
3021 return dev->type == &cxl_dax_region_type;
3022 }
3023
3024 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
3025 {
3026 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
3027 "not a cxl_dax_region device\n"))
3028 return NULL;
3029 return container_of(dev, struct cxl_dax_region, dev);
3030 }
3031 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL");
3032
3033 static struct lock_class_key cxl_dax_region_key;
3034
3035 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
3036 {
3037 struct cxl_region_params *p = &cxlr->params;
3038 struct cxl_dax_region *cxlr_dax;
3039 struct device *dev;
3040
3041 down_read(&cxl_region_rwsem);
3042 if (p->state != CXL_CONFIG_COMMIT) {
3043 cxlr_dax = ERR_PTR(-ENXIO);
3044 goto out;
3045 }
3046
3047 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
3048 if (!cxlr_dax) {
3049 cxlr_dax = ERR_PTR(-ENOMEM);
3050 goto out;
3051 }
3052
3053 cxlr_dax->hpa_range.start = p->res->start;
3054 cxlr_dax->hpa_range.end = p->res->end;
3055
3056 dev = &cxlr_dax->dev;
3057 cxlr_dax->cxlr = cxlr;
3058 device_initialize(dev);
3059 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
3060 device_set_pm_not_required(dev);
3061 dev->parent = &cxlr->dev;
3062 dev->bus = &cxl_bus_type;
3063 dev->type = &cxl_dax_region_type;
3064 out:
3065 up_read(&cxl_region_rwsem);
3066
3067 return cxlr_dax;
3068 }
3069
3070 static void cxlr_pmem_unregister(void *_cxlr_pmem)
3071 {
3072 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
3073 struct cxl_region *cxlr = cxlr_pmem->cxlr;
3074 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3075
3076 /*
3077 * Either the bridge is in ->remove() context under the device_lock(),
3078 * or cxlr_release_nvdimm() is cancelling the bridge's release action
3079 * for @cxlr_pmem and doing it itself (while manually holding the bridge
3080 * lock).
3081 */
3082 device_lock_assert(&cxl_nvb->dev);
3083 cxlr->cxlr_pmem = NULL;
3084 cxlr_pmem->cxlr = NULL;
3085 device_unregister(&cxlr_pmem->dev);
3086 }
3087
3088 static void cxlr_release_nvdimm(void *_cxlr)
3089 {
3090 struct cxl_region *cxlr = _cxlr;
3091 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
3092
3093 scoped_guard(device, &cxl_nvb->dev) {
3094 if (cxlr->cxlr_pmem)
3095 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
3096 cxlr->cxlr_pmem);
3097 }
3098 cxlr->cxl_nvb = NULL;
3099 put_device(&cxl_nvb->dev);
3100 }
3101
3102 /**
3103 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
3104 * @cxlr: parent CXL region for this pmem region bridge device
3105 *
3106 * Return: 0 on success negative error code on failure.
3107 */
3108 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
3109 {
3110 struct cxl_pmem_region *cxlr_pmem;
3111 struct cxl_nvdimm_bridge *cxl_nvb;
3112 struct device *dev;
3113 int rc;
3114
3115 rc = cxl_pmem_region_alloc(cxlr);
3116 if (rc)
3117 return rc;
3118 cxlr_pmem = cxlr->cxlr_pmem;
3119 cxl_nvb = cxlr->cxl_nvb;
3120
3121 dev = &cxlr_pmem->dev;
3122 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
3123 if (rc)
3124 goto err;
3125
3126 rc = device_add(dev);
3127 if (rc)
3128 goto err;
3129
3130 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3131 dev_name(dev));
3132
3133 scoped_guard(device, &cxl_nvb->dev) {
3134 if (cxl_nvb->dev.driver)
3135 rc = devm_add_action_or_reset(&cxl_nvb->dev,
3136 cxlr_pmem_unregister,
3137 cxlr_pmem);
3138 else
3139 rc = -ENXIO;
3140 }
3141
3142 if (rc)
3143 goto err_bridge;
3144
3145 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
3146 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
3147
3148 err:
3149 put_device(dev);
3150 err_bridge:
3151 put_device(&cxl_nvb->dev);
3152 cxlr->cxl_nvb = NULL;
3153 return rc;
3154 }
3155
3156 static void cxlr_dax_unregister(void *_cxlr_dax)
3157 {
3158 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
3159
3160 device_unregister(&cxlr_dax->dev);
3161 }
3162
3163 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
3164 {
3165 struct cxl_dax_region *cxlr_dax;
3166 struct device *dev;
3167 int rc;
3168
3169 cxlr_dax = cxl_dax_region_alloc(cxlr);
3170 if (IS_ERR(cxlr_dax))
3171 return PTR_ERR(cxlr_dax);
3172
3173 dev = &cxlr_dax->dev;
3174 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
3175 if (rc)
3176 goto err;
3177
3178 rc = device_add(dev);
3179 if (rc)
3180 goto err;
3181
3182 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3183 dev_name(dev));
3184
3185 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
3186 cxlr_dax);
3187 err:
3188 put_device(dev);
3189 return rc;
3190 }
3191
3192 static int match_root_decoder_by_range(struct device *dev,
3193 const void *data)
3194 {
3195 const struct range *r1, *r2 = data;
3196 struct cxl_root_decoder *cxlrd;
3197
3198 if (!is_root_decoder(dev))
3199 return 0;
3200
3201 cxlrd = to_cxl_root_decoder(dev);
3202 r1 = &cxlrd->cxlsd.cxld.hpa_range;
3203 return range_contains(r1, r2);
3204 }
3205
3206 static int match_region_by_range(struct device *dev, const void *data)
3207 {
3208 struct cxl_region_params *p;
3209 struct cxl_region *cxlr;
3210 const struct range *r = data;
3211 int rc = 0;
3212
3213 if (!is_cxl_region(dev))
3214 return 0;
3215
3216 cxlr = to_cxl_region(dev);
3217 p = &cxlr->params;
3218
3219 down_read(&cxl_region_rwsem);
3220 if (p->res && p->res->start == r->start && p->res->end == r->end)
3221 rc = 1;
3222 up_read(&cxl_region_rwsem);
3223
3224 return rc;
3225 }
3226
3227 /* Establish an empty region covering the given HPA range */
3228 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
3229 struct cxl_endpoint_decoder *cxled)
3230 {
3231 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3232 struct cxl_port *port = cxlrd_to_port(cxlrd);
3233 struct range *hpa = &cxled->cxld.hpa_range;
3234 struct cxl_region_params *p;
3235 struct cxl_region *cxlr;
3236 struct resource *res;
3237 int rc;
3238
3239 do {
3240 cxlr = __create_region(cxlrd, cxled->mode,
3241 atomic_read(&cxlrd->region_id));
3242 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
3243
3244 if (IS_ERR(cxlr)) {
3245 dev_err(cxlmd->dev.parent,
3246 "%s:%s: %s failed assign region: %ld\n",
3247 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3248 __func__, PTR_ERR(cxlr));
3249 return cxlr;
3250 }
3251
3252 down_write(&cxl_region_rwsem);
3253 p = &cxlr->params;
3254 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3255 dev_err(cxlmd->dev.parent,
3256 "%s:%s: %s autodiscovery interrupted\n",
3257 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3258 __func__);
3259 rc = -EBUSY;
3260 goto err;
3261 }
3262
3263 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
3264
3265 res = kmalloc(sizeof(*res), GFP_KERNEL);
3266 if (!res) {
3267 rc = -ENOMEM;
3268 goto err;
3269 }
3270
3271 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3272 dev_name(&cxlr->dev));
3273 rc = insert_resource(cxlrd->res, res);
3274 if (rc) {
3275 /*
3276 * Platform-firmware may not have split resources like "System
3277 * RAM" on CXL window boundaries see cxl_region_iomem_release()
3278 */
3279 dev_warn(cxlmd->dev.parent,
3280 "%s:%s: %s %s cannot insert resource\n",
3281 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3282 __func__, dev_name(&cxlr->dev));
3283 }
3284
3285 p->res = res;
3286 p->interleave_ways = cxled->cxld.interleave_ways;
3287 p->interleave_granularity = cxled->cxld.interleave_granularity;
3288 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3289
3290 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
3291 if (rc)
3292 goto err;
3293
3294 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3295 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3296 dev_name(&cxlr->dev), p->res, p->interleave_ways,
3297 p->interleave_granularity);
3298
3299 /* ...to match put_device() in cxl_add_to_region() */
3300 get_device(&cxlr->dev);
3301 up_write(&cxl_region_rwsem);
3302
3303 return cxlr;
3304
3305 err:
3306 up_write(&cxl_region_rwsem);
3307 devm_release_action(port->uport_dev, unregister_region, cxlr);
3308 return ERR_PTR(rc);
3309 }
3310
3311 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
3312 {
3313 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3314 struct range *hpa = &cxled->cxld.hpa_range;
3315 struct cxl_decoder *cxld = &cxled->cxld;
3316 struct device *cxlrd_dev, *region_dev;
3317 struct cxl_root_decoder *cxlrd;
3318 struct cxl_region_params *p;
3319 struct cxl_region *cxlr;
3320 bool attach = false;
3321 int rc;
3322
3323 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
3324 match_root_decoder_by_range);
3325 if (!cxlrd_dev) {
3326 dev_err(cxlmd->dev.parent,
3327 "%s:%s no CXL window for range %#llx:%#llx\n",
3328 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3329 cxld->hpa_range.start, cxld->hpa_range.end);
3330 return -ENXIO;
3331 }
3332
3333 cxlrd = to_cxl_root_decoder(cxlrd_dev);
3334
3335 /*
3336 * Ensure that if multiple threads race to construct_region() for @hpa
3337 * one does the construction and the others add to that.
3338 */
3339 mutex_lock(&cxlrd->range_lock);
3340 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3341 match_region_by_range);
3342 if (!region_dev) {
3343 cxlr = construct_region(cxlrd, cxled);
3344 region_dev = &cxlr->dev;
3345 } else
3346 cxlr = to_cxl_region(region_dev);
3347 mutex_unlock(&cxlrd->range_lock);
3348
3349 rc = PTR_ERR_OR_ZERO(cxlr);
3350 if (rc)
3351 goto out;
3352
3353 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3354
3355 down_read(&cxl_region_rwsem);
3356 p = &cxlr->params;
3357 attach = p->state == CXL_CONFIG_COMMIT;
3358 up_read(&cxl_region_rwsem);
3359
3360 if (attach) {
3361 /*
3362 * If device_attach() fails the range may still be active via
3363 * the platform-firmware memory map, otherwise the driver for
3364 * regions is local to this file, so driver matching can't fail.
3365 */
3366 if (device_attach(&cxlr->dev) < 0)
3367 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3368 p->res);
3369 }
3370
3371 put_device(region_dev);
3372 out:
3373 put_device(cxlrd_dev);
3374 return rc;
3375 }
3376 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL");
3377
3378 static int is_system_ram(struct resource *res, void *arg)
3379 {
3380 struct cxl_region *cxlr = arg;
3381 struct cxl_region_params *p = &cxlr->params;
3382
3383 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3384 return 1;
3385 }
3386
3387 static void shutdown_notifiers(void *_cxlr)
3388 {
3389 struct cxl_region *cxlr = _cxlr;
3390
3391 unregister_memory_notifier(&cxlr->memory_notifier);
3392 unregister_mt_adistance_algorithm(&cxlr->adist_notifier);
3393 }
3394
3395 static int cxl_region_probe(struct device *dev)
3396 {
3397 struct cxl_region *cxlr = to_cxl_region(dev);
3398 struct cxl_region_params *p = &cxlr->params;
3399 int rc;
3400
3401 rc = down_read_interruptible(&cxl_region_rwsem);
3402 if (rc) {
3403 dev_dbg(&cxlr->dev, "probe interrupted\n");
3404 return rc;
3405 }
3406
3407 if (p->state < CXL_CONFIG_COMMIT) {
3408 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3409 rc = -ENXIO;
3410 goto out;
3411 }
3412
3413 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3414 dev_err(&cxlr->dev,
3415 "failed to activate, re-commit region and retry\n");
3416 rc = -ENXIO;
3417 goto out;
3418 }
3419
3420 /*
3421 * From this point on any path that changes the region's state away from
3422 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3423 */
3424 out:
3425 up_read(&cxl_region_rwsem);
3426
3427 if (rc)
3428 return rc;
3429
3430 cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
3431 cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
3432 register_memory_notifier(&cxlr->memory_notifier);
3433
3434 cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance;
3435 cxlr->adist_notifier.priority = 100;
3436 register_mt_adistance_algorithm(&cxlr->adist_notifier);
3437
3438 rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr);
3439 if (rc)
3440 return rc;
3441
3442 switch (cxlr->mode) {
3443 case CXL_DECODER_PMEM:
3444 return devm_cxl_add_pmem_region(cxlr);
3445 case CXL_DECODER_RAM:
3446 /*
3447 * The region can not be manged by CXL if any portion of
3448 * it is already online as 'System RAM'
3449 */
3450 if (walk_iomem_res_desc(IORES_DESC_NONE,
3451 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3452 p->res->start, p->res->end, cxlr,
3453 is_system_ram) > 0)
3454 return 0;
3455 return devm_cxl_add_dax_region(cxlr);
3456 default:
3457 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3458 cxlr->mode);
3459 return -ENXIO;
3460 }
3461 }
3462
3463 static struct cxl_driver cxl_region_driver = {
3464 .name = "cxl_region",
3465 .probe = cxl_region_probe,
3466 .id = CXL_DEVICE_REGION,
3467 };
3468
3469 int cxl_region_init(void)
3470 {
3471 return cxl_driver_register(&cxl_region_driver);
3472 }
3473
3474 void cxl_region_exit(void)
3475 {
3476 cxl_driver_unregister(&cxl_region_driver);
3477 }
3478
3479 MODULE_IMPORT_NS("CXL");
3480 MODULE_IMPORT_NS("DEVMEM");
3481 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
Kernel DRM miscellaneous fixes and cross-tree changes