?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hwrng: core - Don't use a stack buffer in add_early_randomness()
[linux/fpc-iii.git] / drivers / acpi / nfit / core.c
blob71a7d07c28c9447d8ed41bc18c7f3ec4dcc83607
1 /*
2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 */
13 #include <linux/list_sort.h>
14 #include <linux/libnvdimm.h>
15 #include <linux/module.h>
16 #include <linux/mutex.h>
17 #include <linux/ndctl.h>
18 #include <linux/sysfs.h>
19 #include <linux/delay.h>
20 #include <linux/list.h>
21 #include <linux/acpi.h>
22 #include <linux/sort.h>
23 #include <linux/pmem.h>
24 #include <linux/io.h>
25 #include <linux/nd.h>
26 #include <asm/cacheflush.h>
27 #include "nfit.h"
28
29 /*
30 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
31 * irrelevant.
32 */
33 #include <linux/io-64-nonatomic-hi-lo.h>
34
35 static bool force_enable_dimms;
36 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
37 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
38
39 static unsigned int scrub_timeout = NFIT_ARS_TIMEOUT;
40 module_param(scrub_timeout, uint, S_IRUGO|S_IWUSR);
41 MODULE_PARM_DESC(scrub_timeout, "Initial scrub timeout in seconds");
42
43 /* after three payloads of overflow, it's dead jim */
44 static unsigned int scrub_overflow_abort = 3;
45 module_param(scrub_overflow_abort, uint, S_IRUGO|S_IWUSR);
46 MODULE_PARM_DESC(scrub_overflow_abort,
47 "Number of times we overflow ARS results before abort");
48
49 static bool disable_vendor_specific;
50 module_param(disable_vendor_specific, bool, S_IRUGO);
51 MODULE_PARM_DESC(disable_vendor_specific,
52 "Limit commands to the publicly specified set\n");
53
54 LIST_HEAD(acpi_descs);
55 DEFINE_MUTEX(acpi_desc_lock);
56
57 static struct workqueue_struct *nfit_wq;
58
59 struct nfit_table_prev {
60 struct list_head spas;
61 struct list_head memdevs;
62 struct list_head dcrs;
63 struct list_head bdws;
64 struct list_head idts;
65 struct list_head flushes;
66 };
67
68 static u8 nfit_uuid[NFIT_UUID_MAX][16];
69
70 const u8 *to_nfit_uuid(enum nfit_uuids id)
71 {
72 return nfit_uuid[id];
73 }
74 EXPORT_SYMBOL(to_nfit_uuid);
75
76 static struct acpi_nfit_desc *to_acpi_nfit_desc(
77 struct nvdimm_bus_descriptor *nd_desc)
78 {
79 return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
80 }
81
82 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
83 {
84 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
85
86 /*
87 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
88 * acpi_device.
89 */
90 if (!nd_desc->provider_name
91 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
92 return NULL;
93
94 return to_acpi_device(acpi_desc->dev);
95 }
96
97 static int xlat_status(void *buf, unsigned int cmd, u32 status)
98 {
99 struct nd_cmd_clear_error *clear_err;
100 struct nd_cmd_ars_status *ars_status;
101 u16 flags;
102
103 switch (cmd) {
104 case ND_CMD_ARS_CAP:
105 if ((status & 0xffff) == NFIT_ARS_CAP_NONE)
106 return -ENOTTY;
107
108 /* Command failed */
109 if (status & 0xffff)
110 return -EIO;
111
112 /* No supported scan types for this range */
113 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
114 if ((status >> 16 & flags) == 0)
115 return -ENOTTY;
116 break;
117 case ND_CMD_ARS_START:
118 /* ARS is in progress */
119 if ((status & 0xffff) == NFIT_ARS_START_BUSY)
120 return -EBUSY;
121
122 /* Command failed */
123 if (status & 0xffff)
124 return -EIO;
125 break;
126 case ND_CMD_ARS_STATUS:
127 ars_status = buf;
128 /* Command failed */
129 if (status & 0xffff)
130 return -EIO;
131 /* Check extended status (Upper two bytes) */
132 if (status == NFIT_ARS_STATUS_DONE)
133 return 0;
134
135 /* ARS is in progress */
136 if (status == NFIT_ARS_STATUS_BUSY)
137 return -EBUSY;
138
139 /* No ARS performed for the current boot */
140 if (status == NFIT_ARS_STATUS_NONE)
141 return -EAGAIN;
142
143 /*
144 * ARS interrupted, either we overflowed or some other
145 * agent wants the scan to stop. If we didn't overflow
146 * then just continue with the returned results.
147 */
148 if (status == NFIT_ARS_STATUS_INTR) {
149 if (ars_status->flags & NFIT_ARS_F_OVERFLOW)
150 return -ENOSPC;
151 return 0;
152 }
153
154 /* Unknown status */
155 if (status >> 16)
156 return -EIO;
157 break;
158 case ND_CMD_CLEAR_ERROR:
159 clear_err = buf;
160 if (status & 0xffff)
161 return -EIO;
162 if (!clear_err->cleared)
163 return -EIO;
164 if (clear_err->length > clear_err->cleared)
165 return clear_err->cleared;
166 break;
167 default:
168 break;
169 }
170
171 /* all other non-zero status results in an error */
172 if (status)
173 return -EIO;
174 return 0;
175 }
176
177 static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
178 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
179 unsigned int buf_len, int *cmd_rc)
180 {
181 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
182 union acpi_object in_obj, in_buf, *out_obj;
183 const struct nd_cmd_desc *desc = NULL;
184 struct device *dev = acpi_desc->dev;
185 struct nd_cmd_pkg *call_pkg = NULL;
186 const char *cmd_name, *dimm_name;
187 unsigned long cmd_mask, dsm_mask;
188 u32 offset, fw_status = 0;
189 acpi_handle handle;
190 unsigned int func;
191 const u8 *uuid;
192 int rc, i;
193
194 func = cmd;
195 if (cmd == ND_CMD_CALL) {
196 call_pkg = buf;
197 func = call_pkg->nd_command;
198 }
199
200 if (nvdimm) {
201 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
202 struct acpi_device *adev = nfit_mem->adev;
203
204 if (!adev)
205 return -ENOTTY;
206 if (call_pkg && nfit_mem->family != call_pkg->nd_family)
207 return -ENOTTY;
208
209 dimm_name = nvdimm_name(nvdimm);
210 cmd_name = nvdimm_cmd_name(cmd);
211 cmd_mask = nvdimm_cmd_mask(nvdimm);
212 dsm_mask = nfit_mem->dsm_mask;
213 desc = nd_cmd_dimm_desc(cmd);
214 uuid = to_nfit_uuid(nfit_mem->family);
215 handle = adev->handle;
216 } else {
217 struct acpi_device *adev = to_acpi_dev(acpi_desc);
218
219 cmd_name = nvdimm_bus_cmd_name(cmd);
220 cmd_mask = nd_desc->cmd_mask;
221 dsm_mask = cmd_mask;
222 desc = nd_cmd_bus_desc(cmd);
223 uuid = to_nfit_uuid(NFIT_DEV_BUS);
224 handle = adev->handle;
225 dimm_name = "bus";
226 }
227
228 if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
229 return -ENOTTY;
230
231 if (!test_bit(cmd, &cmd_mask) || !test_bit(func, &dsm_mask))
232 return -ENOTTY;
233
234 in_obj.type = ACPI_TYPE_PACKAGE;
235 in_obj.package.count = 1;
236 in_obj.package.elements = &in_buf;
237 in_buf.type = ACPI_TYPE_BUFFER;
238 in_buf.buffer.pointer = buf;
239 in_buf.buffer.length = 0;
240
241 /* libnvdimm has already validated the input envelope */
242 for (i = 0; i < desc->in_num; i++)
243 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
244 i, buf);
245
246 if (call_pkg) {
247 /* skip over package wrapper */
248 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
249 in_buf.buffer.length = call_pkg->nd_size_in;
250 }
251
252 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
253 dev_dbg(dev, "%s:%s cmd: %d: func: %d input length: %d\n",
254 __func__, dimm_name, cmd, func,
255 in_buf.buffer.length);
256 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4,
257 in_buf.buffer.pointer,
258 min_t(u32, 256, in_buf.buffer.length), true);
259 }
260
261 out_obj = acpi_evaluate_dsm(handle, uuid, 1, func, &in_obj);
262 if (!out_obj) {
263 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name,
264 cmd_name);
265 return -EINVAL;
266 }
267
268 if (call_pkg) {
269 call_pkg->nd_fw_size = out_obj->buffer.length;
270 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
271 out_obj->buffer.pointer,
272 min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
273
274 ACPI_FREE(out_obj);
275 /*
276 * Need to support FW function w/o known size in advance.
277 * Caller can determine required size based upon nd_fw_size.
278 * If we return an error (like elsewhere) then caller wouldn't
279 * be able to rely upon data returned to make calculation.
280 */
281 return 0;
282 }
283
284 if (out_obj->package.type != ACPI_TYPE_BUFFER) {
285 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n",
286 __func__, dimm_name, cmd_name, out_obj->type);
287 rc = -EINVAL;
288 goto out;
289 }
290
291 if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
292 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__,
293 dimm_name, cmd_name, out_obj->buffer.length);
294 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
295 4, out_obj->buffer.pointer, min_t(u32, 128,
296 out_obj->buffer.length), true);
297 }
298
299 for (i = 0, offset = 0; i < desc->out_num; i++) {
300 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
301 (u32 *) out_obj->buffer.pointer);
302
303 if (offset + out_size > out_obj->buffer.length) {
304 dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n",
305 __func__, dimm_name, cmd_name, i);
306 break;
307 }
308
309 if (in_buf.buffer.length + offset + out_size > buf_len) {
310 dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n",
311 __func__, dimm_name, cmd_name, i);
312 rc = -ENXIO;
313 goto out;
314 }
315 memcpy(buf + in_buf.buffer.length + offset,
316 out_obj->buffer.pointer + offset, out_size);
317 offset += out_size;
318 }
319
320 /*
321 * Set fw_status for all the commands with a known format to be
322 * later interpreted by xlat_status().
323 */
324 if (i >= 1 && ((cmd >= ND_CMD_ARS_CAP && cmd <= ND_CMD_CLEAR_ERROR)
325 || (cmd >= ND_CMD_SMART && cmd <= ND_CMD_VENDOR)))
326 fw_status = *(u32 *) out_obj->buffer.pointer;
327
328 if (offset + in_buf.buffer.length < buf_len) {
329 if (i >= 1) {
330 /*
331 * status valid, return the number of bytes left
332 * unfilled in the output buffer
333 */
334 rc = buf_len - offset - in_buf.buffer.length;
335 if (cmd_rc)
336 *cmd_rc = xlat_status(buf, cmd, fw_status);
337 } else {
338 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
339 __func__, dimm_name, cmd_name, buf_len,
340 offset);
341 rc = -ENXIO;
342 }
343 } else {
344 rc = 0;
345 if (cmd_rc)
346 *cmd_rc = xlat_status(buf, cmd, fw_status);
347 }
348
349 out:
350 ACPI_FREE(out_obj);
351
352 return rc;
353 }
354
355 static const char *spa_type_name(u16 type)
356 {
357 static const char *to_name[] = {
358 [NFIT_SPA_VOLATILE] = "volatile",
359 [NFIT_SPA_PM] = "pmem",
360 [NFIT_SPA_DCR] = "dimm-control-region",
361 [NFIT_SPA_BDW] = "block-data-window",
362 [NFIT_SPA_VDISK] = "volatile-disk",
363 [NFIT_SPA_VCD] = "volatile-cd",
364 [NFIT_SPA_PDISK] = "persistent-disk",
365 [NFIT_SPA_PCD] = "persistent-cd",
366
367 };
368
369 if (type > NFIT_SPA_PCD)
370 return "unknown";
371
372 return to_name[type];
373 }
374
375 int nfit_spa_type(struct acpi_nfit_system_address *spa)
376 {
377 int i;
378
379 for (i = 0; i < NFIT_UUID_MAX; i++)
380 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0)
381 return i;
382 return -1;
383 }
384
385 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
386 struct nfit_table_prev *prev,
387 struct acpi_nfit_system_address *spa)
388 {
389 struct device *dev = acpi_desc->dev;
390 struct nfit_spa *nfit_spa;
391
392 if (spa->header.length != sizeof(*spa))
393 return false;
394
395 list_for_each_entry(nfit_spa, &prev->spas, list) {
396 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) {
397 list_move_tail(&nfit_spa->list, &acpi_desc->spas);
398 return true;
399 }
400 }
401
402 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa),
403 GFP_KERNEL);
404 if (!nfit_spa)
405 return false;
406 INIT_LIST_HEAD(&nfit_spa->list);
407 memcpy(nfit_spa->spa, spa, sizeof(*spa));
408 list_add_tail(&nfit_spa->list, &acpi_desc->spas);
409 dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__,
410 spa->range_index,
411 spa_type_name(nfit_spa_type(spa)));
412 return true;
413 }
414
415 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
416 struct nfit_table_prev *prev,
417 struct acpi_nfit_memory_map *memdev)
418 {
419 struct device *dev = acpi_desc->dev;
420 struct nfit_memdev *nfit_memdev;
421
422 if (memdev->header.length != sizeof(*memdev))
423 return false;
424
425 list_for_each_entry(nfit_memdev, &prev->memdevs, list)
426 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) {
427 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
428 return true;
429 }
430
431 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev),
432 GFP_KERNEL);
433 if (!nfit_memdev)
434 return false;
435 INIT_LIST_HEAD(&nfit_memdev->list);
436 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev));
437 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
438 dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n",
439 __func__, memdev->device_handle, memdev->range_index,
440 memdev->region_index);
441 return true;
442 }
443
444 /*
445 * An implementation may provide a truncated control region if no block windows
446 * are defined.
447 */
448 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr)
449 {
450 if (dcr->header.length < offsetof(struct acpi_nfit_control_region,
451 window_size))
452 return 0;
453 if (dcr->windows)
454 return sizeof(*dcr);
455 return offsetof(struct acpi_nfit_control_region, window_size);
456 }
457
458 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
459 struct nfit_table_prev *prev,
460 struct acpi_nfit_control_region *dcr)
461 {
462 struct device *dev = acpi_desc->dev;
463 struct nfit_dcr *nfit_dcr;
464
465 if (!sizeof_dcr(dcr))
466 return false;
467
468 list_for_each_entry(nfit_dcr, &prev->dcrs, list)
469 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) {
470 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
471 return true;
472 }
473
474 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr),
475 GFP_KERNEL);
476 if (!nfit_dcr)
477 return false;
478 INIT_LIST_HEAD(&nfit_dcr->list);
479 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr));
480 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
481 dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__,
482 dcr->region_index, dcr->windows);
483 return true;
484 }
485
486 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
487 struct nfit_table_prev *prev,
488 struct acpi_nfit_data_region *bdw)
489 {
490 struct device *dev = acpi_desc->dev;
491 struct nfit_bdw *nfit_bdw;
492
493 if (bdw->header.length != sizeof(*bdw))
494 return false;
495 list_for_each_entry(nfit_bdw, &prev->bdws, list)
496 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) {
497 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
498 return true;
499 }
500
501 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw),
502 GFP_KERNEL);
503 if (!nfit_bdw)
504 return false;
505 INIT_LIST_HEAD(&nfit_bdw->list);
506 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw));
507 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
508 dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__,
509 bdw->region_index, bdw->windows);
510 return true;
511 }
512
513 static size_t sizeof_idt(struct acpi_nfit_interleave *idt)
514 {
515 if (idt->header.length < sizeof(*idt))
516 return 0;
517 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1);
518 }
519
520 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
521 struct nfit_table_prev *prev,
522 struct acpi_nfit_interleave *idt)
523 {
524 struct device *dev = acpi_desc->dev;
525 struct nfit_idt *nfit_idt;
526
527 if (!sizeof_idt(idt))
528 return false;
529
530 list_for_each_entry(nfit_idt, &prev->idts, list) {
531 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt))
532 continue;
533
534 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) {
535 list_move_tail(&nfit_idt->list, &acpi_desc->idts);
536 return true;
537 }
538 }
539
540 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt),
541 GFP_KERNEL);
542 if (!nfit_idt)
543 return false;
544 INIT_LIST_HEAD(&nfit_idt->list);
545 memcpy(nfit_idt->idt, idt, sizeof_idt(idt));
546 list_add_tail(&nfit_idt->list, &acpi_desc->idts);
547 dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__,
548 idt->interleave_index, idt->line_count);
549 return true;
550 }
551
552 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush)
553 {
554 if (flush->header.length < sizeof(*flush))
555 return 0;
556 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1);
557 }
558
559 static bool add_flush(struct acpi_nfit_desc *acpi_desc,
560 struct nfit_table_prev *prev,
561 struct acpi_nfit_flush_address *flush)
562 {
563 struct device *dev = acpi_desc->dev;
564 struct nfit_flush *nfit_flush;
565
566 if (!sizeof_flush(flush))
567 return false;
568
569 list_for_each_entry(nfit_flush, &prev->flushes, list) {
570 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush))
571 continue;
572
573 if (memcmp(nfit_flush->flush, flush,
574 sizeof_flush(flush)) == 0) {
575 list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
576 return true;
577 }
578 }
579
580 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush)
581 + sizeof_flush(flush), GFP_KERNEL);
582 if (!nfit_flush)
583 return false;
584 INIT_LIST_HEAD(&nfit_flush->list);
585 memcpy(nfit_flush->flush, flush, sizeof_flush(flush));
586 list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
587 dev_dbg(dev, "%s: nfit_flush handle: %d hint_count: %d\n", __func__,
588 flush->device_handle, flush->hint_count);
589 return true;
590 }
591
592 static void *add_table(struct acpi_nfit_desc *acpi_desc,
593 struct nfit_table_prev *prev, void *table, const void *end)
594 {
595 struct device *dev = acpi_desc->dev;
596 struct acpi_nfit_header *hdr;
597 void *err = ERR_PTR(-ENOMEM);
598
599 if (table >= end)
600 return NULL;
601
602 hdr = table;
603 if (!hdr->length) {
604 dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
605 hdr->type);
606 return NULL;
607 }
608
609 switch (hdr->type) {
610 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
611 if (!add_spa(acpi_desc, prev, table))
612 return err;
613 break;
614 case ACPI_NFIT_TYPE_MEMORY_MAP:
615 if (!add_memdev(acpi_desc, prev, table))
616 return err;
617 break;
618 case ACPI_NFIT_TYPE_CONTROL_REGION:
619 if (!add_dcr(acpi_desc, prev, table))
620 return err;
621 break;
622 case ACPI_NFIT_TYPE_DATA_REGION:
623 if (!add_bdw(acpi_desc, prev, table))
624 return err;
625 break;
626 case ACPI_NFIT_TYPE_INTERLEAVE:
627 if (!add_idt(acpi_desc, prev, table))
628 return err;
629 break;
630 case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
631 if (!add_flush(acpi_desc, prev, table))
632 return err;
633 break;
634 case ACPI_NFIT_TYPE_SMBIOS:
635 dev_dbg(dev, "%s: smbios\n", __func__);
636 break;
637 default:
638 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
639 break;
640 }
641
642 return table + hdr->length;
643 }
644
645 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
646 struct nfit_mem *nfit_mem)
647 {
648 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
649 u16 dcr = nfit_mem->dcr->region_index;
650 struct nfit_spa *nfit_spa;
651
652 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
653 u16 range_index = nfit_spa->spa->range_index;
654 int type = nfit_spa_type(nfit_spa->spa);
655 struct nfit_memdev *nfit_memdev;
656
657 if (type != NFIT_SPA_BDW)
658 continue;
659
660 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
661 if (nfit_memdev->memdev->range_index != range_index)
662 continue;
663 if (nfit_memdev->memdev->device_handle != device_handle)
664 continue;
665 if (nfit_memdev->memdev->region_index != dcr)
666 continue;
667
668 nfit_mem->spa_bdw = nfit_spa->spa;
669 return;
670 }
671 }
672
673 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
674 nfit_mem->spa_dcr->range_index);
675 nfit_mem->bdw = NULL;
676 }
677
678 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
679 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
680 {
681 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
682 struct nfit_memdev *nfit_memdev;
683 struct nfit_bdw *nfit_bdw;
684 struct nfit_idt *nfit_idt;
685 u16 idt_idx, range_index;
686
687 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
688 if (nfit_bdw->bdw->region_index != dcr)
689 continue;
690 nfit_mem->bdw = nfit_bdw->bdw;
691 break;
692 }
693
694 if (!nfit_mem->bdw)
695 return;
696
697 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
698
699 if (!nfit_mem->spa_bdw)
700 return;
701
702 range_index = nfit_mem->spa_bdw->range_index;
703 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
704 if (nfit_memdev->memdev->range_index != range_index ||
705 nfit_memdev->memdev->region_index != dcr)
706 continue;
707 nfit_mem->memdev_bdw = nfit_memdev->memdev;
708 idt_idx = nfit_memdev->memdev->interleave_index;
709 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
710 if (nfit_idt->idt->interleave_index != idt_idx)
711 continue;
712 nfit_mem->idt_bdw = nfit_idt->idt;
713 break;
714 }
715 break;
716 }
717 }
718
719 static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
720 struct acpi_nfit_system_address *spa)
721 {
722 struct nfit_mem *nfit_mem, *found;
723 struct nfit_memdev *nfit_memdev;
724 int type = nfit_spa_type(spa);
725
726 switch (type) {
727 case NFIT_SPA_DCR:
728 case NFIT_SPA_PM:
729 break;
730 default:
731 return 0;
732 }
733
734 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
735 struct nfit_flush *nfit_flush;
736 struct nfit_dcr *nfit_dcr;
737 u32 device_handle;
738 u16 dcr;
739
740 if (nfit_memdev->memdev->range_index != spa->range_index)
741 continue;
742 found = NULL;
743 dcr = nfit_memdev->memdev->region_index;
744 device_handle = nfit_memdev->memdev->device_handle;
745 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
746 if (__to_nfit_memdev(nfit_mem)->device_handle
747 == device_handle) {
748 found = nfit_mem;
749 break;
750 }
751
752 if (found)
753 nfit_mem = found;
754 else {
755 nfit_mem = devm_kzalloc(acpi_desc->dev,
756 sizeof(*nfit_mem), GFP_KERNEL);
757 if (!nfit_mem)
758 return -ENOMEM;
759 INIT_LIST_HEAD(&nfit_mem->list);
760 nfit_mem->acpi_desc = acpi_desc;
761 list_add(&nfit_mem->list, &acpi_desc->dimms);
762 }
763
764 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
765 if (nfit_dcr->dcr->region_index != dcr)
766 continue;
767 /*
768 * Record the control region for the dimm. For
769 * the ACPI 6.1 case, where there are separate
770 * control regions for the pmem vs blk
771 * interfaces, be sure to record the extended
772 * blk details.
773 */
774 if (!nfit_mem->dcr)
775 nfit_mem->dcr = nfit_dcr->dcr;
776 else if (nfit_mem->dcr->windows == 0
777 && nfit_dcr->dcr->windows)
778 nfit_mem->dcr = nfit_dcr->dcr;
779 break;
780 }
781
782 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
783 struct acpi_nfit_flush_address *flush;
784 u16 i;
785
786 if (nfit_flush->flush->device_handle != device_handle)
787 continue;
788 nfit_mem->nfit_flush = nfit_flush;
789 flush = nfit_flush->flush;
790 nfit_mem->flush_wpq = devm_kzalloc(acpi_desc->dev,
791 flush->hint_count
792 * sizeof(struct resource), GFP_KERNEL);
793 if (!nfit_mem->flush_wpq)
794 return -ENOMEM;
795 for (i = 0; i < flush->hint_count; i++) {
796 struct resource *res = &nfit_mem->flush_wpq[i];
797
798 res->start = flush->hint_address[i];
799 res->end = res->start + 8 - 1;
800 }
801 break;
802 }
803
804 if (dcr && !nfit_mem->dcr) {
805 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
806 spa->range_index, dcr);
807 return -ENODEV;
808 }
809
810 if (type == NFIT_SPA_DCR) {
811 struct nfit_idt *nfit_idt;
812 u16 idt_idx;
813
814 /* multiple dimms may share a SPA when interleaved */
815 nfit_mem->spa_dcr = spa;
816 nfit_mem->memdev_dcr = nfit_memdev->memdev;
817 idt_idx = nfit_memdev->memdev->interleave_index;
818 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
819 if (nfit_idt->idt->interleave_index != idt_idx)
820 continue;
821 nfit_mem->idt_dcr = nfit_idt->idt;
822 break;
823 }
824 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
825 } else {
826 /*
827 * A single dimm may belong to multiple SPA-PM
828 * ranges, record at least one in addition to
829 * any SPA-DCR range.
830 */
831 nfit_mem->memdev_pmem = nfit_memdev->memdev;
832 }
833 }
834
835 return 0;
836 }
837
838 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
839 {
840 struct nfit_mem *a = container_of(_a, typeof(*a), list);
841 struct nfit_mem *b = container_of(_b, typeof(*b), list);
842 u32 handleA, handleB;
843
844 handleA = __to_nfit_memdev(a)->device_handle;
845 handleB = __to_nfit_memdev(b)->device_handle;
846 if (handleA < handleB)
847 return -1;
848 else if (handleA > handleB)
849 return 1;
850 return 0;
851 }
852
853 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
854 {
855 struct nfit_spa *nfit_spa;
856
857 /*
858 * For each SPA-DCR or SPA-PMEM address range find its
859 * corresponding MEMDEV(s). From each MEMDEV find the
860 * corresponding DCR. Then, if we're operating on a SPA-DCR,
861 * try to find a SPA-BDW and a corresponding BDW that references
862 * the DCR. Throw it all into an nfit_mem object. Note, that
863 * BDWs are optional.
864 */
865 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
866 int rc;
867
868 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa);
869 if (rc)
870 return rc;
871 }
872
873 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
874
875 return 0;
876 }
877
878 static ssize_t revision_show(struct device *dev,
879 struct device_attribute *attr, char *buf)
880 {
881 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
882 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
883 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
884
885 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
886 }
887 static DEVICE_ATTR_RO(revision);
888
889 static ssize_t hw_error_scrub_show(struct device *dev,
890 struct device_attribute *attr, char *buf)
891 {
892 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
893 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
894 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
895
896 return sprintf(buf, "%d\n", acpi_desc->scrub_mode);
897 }
898
899 /*
900 * The 'hw_error_scrub' attribute can have the following values written to it:
901 * '0': Switch to the default mode where an exception will only insert
902 * the address of the memory error into the poison and badblocks lists.
903 * '1': Enable a full scrub to happen if an exception for a memory error is
904 * received.
905 */
906 static ssize_t hw_error_scrub_store(struct device *dev,
907 struct device_attribute *attr, const char *buf, size_t size)
908 {
909 struct nvdimm_bus_descriptor *nd_desc;
910 ssize_t rc;
911 long val;
912
913 rc = kstrtol(buf, 0, &val);
914 if (rc)
915 return rc;
916
917 device_lock(dev);
918 nd_desc = dev_get_drvdata(dev);
919 if (nd_desc) {
920 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
921
922 switch (val) {
923 case HW_ERROR_SCRUB_ON:
924 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON;
925 break;
926 case HW_ERROR_SCRUB_OFF:
927 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF;
928 break;
929 default:
930 rc = -EINVAL;
931 break;
932 }
933 }
934 device_unlock(dev);
935 if (rc)
936 return rc;
937 return size;
938 }
939 static DEVICE_ATTR_RW(hw_error_scrub);
940
941 /*
942 * This shows the number of full Address Range Scrubs that have been
943 * completed since driver load time. Userspace can wait on this using
944 * select/poll etc. A '+' at the end indicates an ARS is in progress
945 */
946 static ssize_t scrub_show(struct device *dev,
947 struct device_attribute *attr, char *buf)
948 {
949 struct nvdimm_bus_descriptor *nd_desc;
950 ssize_t rc = -ENXIO;
951
952 device_lock(dev);
953 nd_desc = dev_get_drvdata(dev);
954 if (nd_desc) {
955 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
956
957 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count,
958 (work_busy(&acpi_desc->work)) ? "+\n" : "\n");
959 }
960 device_unlock(dev);
961 return rc;
962 }
963
964 static ssize_t scrub_store(struct device *dev,
965 struct device_attribute *attr, const char *buf, size_t size)
966 {
967 struct nvdimm_bus_descriptor *nd_desc;
968 ssize_t rc;
969 long val;
970
971 rc = kstrtol(buf, 0, &val);
972 if (rc)
973 return rc;
974 if (val != 1)
975 return -EINVAL;
976
977 device_lock(dev);
978 nd_desc = dev_get_drvdata(dev);
979 if (nd_desc) {
980 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
981
982 rc = acpi_nfit_ars_rescan(acpi_desc);
983 }
984 device_unlock(dev);
985 if (rc)
986 return rc;
987 return size;
988 }
989 static DEVICE_ATTR_RW(scrub);
990
991 static bool ars_supported(struct nvdimm_bus *nvdimm_bus)
992 {
993 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
994 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START
995 | 1 << ND_CMD_ARS_STATUS;
996
997 return (nd_desc->cmd_mask & mask) == mask;
998 }
999
1000 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n)
1001 {
1002 struct device *dev = container_of(kobj, struct device, kobj);
1003 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1004
1005 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus))
1006 return 0;
1007 return a->mode;
1008 }
1009
1010 static struct attribute *acpi_nfit_attributes[] = {
1011 &dev_attr_revision.attr,
1012 &dev_attr_scrub.attr,
1013 &dev_attr_hw_error_scrub.attr,
1014 NULL,
1015 };
1016
1017 static struct attribute_group acpi_nfit_attribute_group = {
1018 .name = "nfit",
1019 .attrs = acpi_nfit_attributes,
1020 .is_visible = nfit_visible,
1021 };
1022
1023 static const struct attribute_group *acpi_nfit_attribute_groups[] = {
1024 &nvdimm_bus_attribute_group,
1025 &acpi_nfit_attribute_group,
1026 NULL,
1027 };
1028
1029 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
1030 {
1031 struct nvdimm *nvdimm = to_nvdimm(dev);
1032 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1033
1034 return __to_nfit_memdev(nfit_mem);
1035 }
1036
1037 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
1038 {
1039 struct nvdimm *nvdimm = to_nvdimm(dev);
1040 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1041
1042 return nfit_mem->dcr;
1043 }
1044
1045 static ssize_t handle_show(struct device *dev,
1046 struct device_attribute *attr, char *buf)
1047 {
1048 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1049
1050 return sprintf(buf, "%#x\n", memdev->device_handle);
1051 }
1052 static DEVICE_ATTR_RO(handle);
1053
1054 static ssize_t phys_id_show(struct device *dev,
1055 struct device_attribute *attr, char *buf)
1056 {
1057 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1058
1059 return sprintf(buf, "%#x\n", memdev->physical_id);
1060 }
1061 static DEVICE_ATTR_RO(phys_id);
1062
1063 static ssize_t vendor_show(struct device *dev,
1064 struct device_attribute *attr, char *buf)
1065 {
1066 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1067
1068 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id));
1069 }
1070 static DEVICE_ATTR_RO(vendor);
1071
1072 static ssize_t rev_id_show(struct device *dev,
1073 struct device_attribute *attr, char *buf)
1074 {
1075 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1076
1077 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id));
1078 }
1079 static DEVICE_ATTR_RO(rev_id);
1080
1081 static ssize_t device_show(struct device *dev,
1082 struct device_attribute *attr, char *buf)
1083 {
1084 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1085
1086 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id));
1087 }
1088 static DEVICE_ATTR_RO(device);
1089
1090 static ssize_t subsystem_vendor_show(struct device *dev,
1091 struct device_attribute *attr, char *buf)
1092 {
1093 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1094
1095 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id));
1096 }
1097 static DEVICE_ATTR_RO(subsystem_vendor);
1098
1099 static ssize_t subsystem_rev_id_show(struct device *dev,
1100 struct device_attribute *attr, char *buf)
1101 {
1102 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1103
1104 return sprintf(buf, "0x%04x\n",
1105 be16_to_cpu(dcr->subsystem_revision_id));
1106 }
1107 static DEVICE_ATTR_RO(subsystem_rev_id);
1108
1109 static ssize_t subsystem_device_show(struct device *dev,
1110 struct device_attribute *attr, char *buf)
1111 {
1112 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1113
1114 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id));
1115 }
1116 static DEVICE_ATTR_RO(subsystem_device);
1117
1118 static int num_nvdimm_formats(struct nvdimm *nvdimm)
1119 {
1120 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1121 int formats = 0;
1122
1123 if (nfit_mem->memdev_pmem)
1124 formats++;
1125 if (nfit_mem->memdev_bdw)
1126 formats++;
1127 return formats;
1128 }
1129
1130 static ssize_t format_show(struct device *dev,
1131 struct device_attribute *attr, char *buf)
1132 {
1133 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1134
1135 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code));
1136 }
1137 static DEVICE_ATTR_RO(format);
1138
1139 static ssize_t format1_show(struct device *dev,
1140 struct device_attribute *attr, char *buf)
1141 {
1142 u32 handle;
1143 ssize_t rc = -ENXIO;
1144 struct nfit_mem *nfit_mem;
1145 struct nfit_memdev *nfit_memdev;
1146 struct acpi_nfit_desc *acpi_desc;
1147 struct nvdimm *nvdimm = to_nvdimm(dev);
1148 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1149
1150 nfit_mem = nvdimm_provider_data(nvdimm);
1151 acpi_desc = nfit_mem->acpi_desc;
1152 handle = to_nfit_memdev(dev)->device_handle;
1153
1154 /* assumes DIMMs have at most 2 published interface codes */
1155 mutex_lock(&acpi_desc->init_mutex);
1156 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1157 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1158 struct nfit_dcr *nfit_dcr;
1159
1160 if (memdev->device_handle != handle)
1161 continue;
1162
1163 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1164 if (nfit_dcr->dcr->region_index != memdev->region_index)
1165 continue;
1166 if (nfit_dcr->dcr->code == dcr->code)
1167 continue;
1168 rc = sprintf(buf, "0x%04x\n",
1169 le16_to_cpu(nfit_dcr->dcr->code));
1170 break;
1171 }
1172 if (rc != ENXIO)
1173 break;
1174 }
1175 mutex_unlock(&acpi_desc->init_mutex);
1176 return rc;
1177 }
1178 static DEVICE_ATTR_RO(format1);
1179
1180 static ssize_t formats_show(struct device *dev,
1181 struct device_attribute *attr, char *buf)
1182 {
1183 struct nvdimm *nvdimm = to_nvdimm(dev);
1184
1185 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm));
1186 }
1187 static DEVICE_ATTR_RO(formats);
1188
1189 static ssize_t serial_show(struct device *dev,
1190 struct device_attribute *attr, char *buf)
1191 {
1192 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1193
1194 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number));
1195 }
1196 static DEVICE_ATTR_RO(serial);
1197
1198 static ssize_t family_show(struct device *dev,
1199 struct device_attribute *attr, char *buf)
1200 {
1201 struct nvdimm *nvdimm = to_nvdimm(dev);
1202 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1203
1204 if (nfit_mem->family < 0)
1205 return -ENXIO;
1206 return sprintf(buf, "%d\n", nfit_mem->family);
1207 }
1208 static DEVICE_ATTR_RO(family);
1209
1210 static ssize_t dsm_mask_show(struct device *dev,
1211 struct device_attribute *attr, char *buf)
1212 {
1213 struct nvdimm *nvdimm = to_nvdimm(dev);
1214 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1215
1216 if (nfit_mem->family < 0)
1217 return -ENXIO;
1218 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask);
1219 }
1220 static DEVICE_ATTR_RO(dsm_mask);
1221
1222 static ssize_t flags_show(struct device *dev,
1223 struct device_attribute *attr, char *buf)
1224 {
1225 u16 flags = to_nfit_memdev(dev)->flags;
1226
1227 return sprintf(buf, "%s%s%s%s%s\n",
1228 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
1229 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
1230 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
1231 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
1232 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "");
1233 }
1234 static DEVICE_ATTR_RO(flags);
1235
1236 static ssize_t id_show(struct device *dev,
1237 struct device_attribute *attr, char *buf)
1238 {
1239 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1240
1241 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID)
1242 return sprintf(buf, "%04x-%02x-%04x-%08x\n",
1243 be16_to_cpu(dcr->vendor_id),
1244 dcr->manufacturing_location,
1245 be16_to_cpu(dcr->manufacturing_date),
1246 be32_to_cpu(dcr->serial_number));
1247 else
1248 return sprintf(buf, "%04x-%08x\n",
1249 be16_to_cpu(dcr->vendor_id),
1250 be32_to_cpu(dcr->serial_number));
1251 }
1252 static DEVICE_ATTR_RO(id);
1253
1254 static struct attribute *acpi_nfit_dimm_attributes[] = {
1255 &dev_attr_handle.attr,
1256 &dev_attr_phys_id.attr,
1257 &dev_attr_vendor.attr,
1258 &dev_attr_device.attr,
1259 &dev_attr_rev_id.attr,
1260 &dev_attr_subsystem_vendor.attr,
1261 &dev_attr_subsystem_device.attr,
1262 &dev_attr_subsystem_rev_id.attr,
1263 &dev_attr_format.attr,
1264 &dev_attr_formats.attr,
1265 &dev_attr_format1.attr,
1266 &dev_attr_serial.attr,
1267 &dev_attr_flags.attr,
1268 &dev_attr_id.attr,
1269 &dev_attr_family.attr,
1270 &dev_attr_dsm_mask.attr,
1271 NULL,
1272 };
1273
1274 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
1275 struct attribute *a, int n)
1276 {
1277 struct device *dev = container_of(kobj, struct device, kobj);
1278 struct nvdimm *nvdimm = to_nvdimm(dev);
1279
1280 if (!to_nfit_dcr(dev))
1281 return 0;
1282 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1)
1283 return 0;
1284 return a->mode;
1285 }
1286
1287 static struct attribute_group acpi_nfit_dimm_attribute_group = {
1288 .name = "nfit",
1289 .attrs = acpi_nfit_dimm_attributes,
1290 .is_visible = acpi_nfit_dimm_attr_visible,
1291 };
1292
1293 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
1294 &nvdimm_attribute_group,
1295 &nd_device_attribute_group,
1296 &acpi_nfit_dimm_attribute_group,
1297 NULL,
1298 };
1299
1300 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
1301 u32 device_handle)
1302 {
1303 struct nfit_mem *nfit_mem;
1304
1305 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1306 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
1307 return nfit_mem->nvdimm;
1308
1309 return NULL;
1310 }
1311
1312 void __acpi_nvdimm_notify(struct device *dev, u32 event)
1313 {
1314 struct nfit_mem *nfit_mem;
1315 struct acpi_nfit_desc *acpi_desc;
1316
1317 dev_dbg(dev->parent, "%s: %s: event: %d\n", dev_name(dev), __func__,
1318 event);
1319
1320 if (event != NFIT_NOTIFY_DIMM_HEALTH) {
1321 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev),
1322 event);
1323 return;
1324 }
1325
1326 acpi_desc = dev_get_drvdata(dev->parent);
1327 if (!acpi_desc)
1328 return;
1329
1330 /*
1331 * If we successfully retrieved acpi_desc, then we know nfit_mem data
1332 * is still valid.
1333 */
1334 nfit_mem = dev_get_drvdata(dev);
1335 if (nfit_mem && nfit_mem->flags_attr)
1336 sysfs_notify_dirent(nfit_mem->flags_attr);
1337 }
1338 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify);
1339
1340 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
1341 {
1342 struct acpi_device *adev = data;
1343 struct device *dev = &adev->dev;
1344
1345 device_lock(dev->parent);
1346 __acpi_nvdimm_notify(dev, event);
1347 device_unlock(dev->parent);
1348 }
1349
1350 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
1351 struct nfit_mem *nfit_mem, u32 device_handle)
1352 {
1353 struct acpi_device *adev, *adev_dimm;
1354 struct device *dev = acpi_desc->dev;
1355 unsigned long dsm_mask;
1356 const u8 *uuid;
1357 int i;
1358
1359 /* nfit test assumes 1:1 relationship between commands and dsms */
1360 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
1361 nfit_mem->family = NVDIMM_FAMILY_INTEL;
1362 adev = to_acpi_dev(acpi_desc);
1363 if (!adev)
1364 return 0;
1365
1366 adev_dimm = acpi_find_child_device(adev, device_handle, false);
1367 nfit_mem->adev = adev_dimm;
1368 if (!adev_dimm) {
1369 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
1370 device_handle);
1371 return force_enable_dimms ? 0 : -ENODEV;
1372 }
1373
1374 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle,
1375 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) {
1376 dev_err(dev, "%s: notification registration failed\n",
1377 dev_name(&adev_dimm->dev));
1378 return -ENXIO;
1379 }
1380
1381 /*
1382 * Until standardization materializes we need to consider 4
1383 * different command sets. Note, that checking for function0 (bit0)
1384 * tells us if any commands are reachable through this uuid.
1385 */
1386 for (i = NVDIMM_FAMILY_INTEL; i <= NVDIMM_FAMILY_MSFT; i++)
1387 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1))
1388 break;
1389
1390 /* limit the supported commands to those that are publicly documented */
1391 nfit_mem->family = i;
1392 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
1393 dsm_mask = 0x3fe;
1394 if (disable_vendor_specific)
1395 dsm_mask &= ~(1 << ND_CMD_VENDOR);
1396 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) {
1397 dsm_mask = 0x1c3c76;
1398 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) {
1399 dsm_mask = 0x1fe;
1400 if (disable_vendor_specific)
1401 dsm_mask &= ~(1 << 8);
1402 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) {
1403 dsm_mask = 0xffffffff;
1404 } else {
1405 dev_dbg(dev, "unknown dimm command family\n");
1406 nfit_mem->family = -1;
1407 /* DSMs are optional, continue loading the driver... */
1408 return 0;
1409 }
1410
1411 uuid = to_nfit_uuid(nfit_mem->family);
1412 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
1413 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i))
1414 set_bit(i, &nfit_mem->dsm_mask);
1415
1416 return 0;
1417 }
1418
1419 static void shutdown_dimm_notify(void *data)
1420 {
1421 struct acpi_nfit_desc *acpi_desc = data;
1422 struct nfit_mem *nfit_mem;
1423
1424 mutex_lock(&acpi_desc->init_mutex);
1425 /*
1426 * Clear out the nfit_mem->flags_attr and shut down dimm event
1427 * notifications.
1428 */
1429 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1430 struct acpi_device *adev_dimm = nfit_mem->adev;
1431
1432 if (nfit_mem->flags_attr) {
1433 sysfs_put(nfit_mem->flags_attr);
1434 nfit_mem->flags_attr = NULL;
1435 }
1436 if (adev_dimm)
1437 acpi_remove_notify_handler(adev_dimm->handle,
1438 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify);
1439 }
1440 mutex_unlock(&acpi_desc->init_mutex);
1441 }
1442
1443 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
1444 {
1445 struct nfit_mem *nfit_mem;
1446 int dimm_count = 0, rc;
1447 struct nvdimm *nvdimm;
1448
1449 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1450 struct acpi_nfit_flush_address *flush;
1451 unsigned long flags = 0, cmd_mask;
1452 u32 device_handle;
1453 u16 mem_flags;
1454
1455 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
1456 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
1457 if (nvdimm) {
1458 dimm_count++;
1459 continue;
1460 }
1461
1462 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
1463 flags |= NDD_ALIASING;
1464
1465 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
1466 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
1467 flags |= NDD_UNARMED;
1468
1469 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
1470 if (rc)
1471 continue;
1472
1473 /*
1474 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
1475 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
1476 * userspace interface.
1477 */
1478 cmd_mask = 1UL << ND_CMD_CALL;
1479 if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
1480 cmd_mask |= nfit_mem->dsm_mask;
1481
1482 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush
1483 : NULL;
1484 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
1485 acpi_nfit_dimm_attribute_groups,
1486 flags, cmd_mask, flush ? flush->hint_count : 0,
1487 nfit_mem->flush_wpq);
1488 if (!nvdimm)
1489 return -ENOMEM;
1490
1491 nfit_mem->nvdimm = nvdimm;
1492 dimm_count++;
1493
1494 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
1495 continue;
1496
1497 dev_info(acpi_desc->dev, "%s flags:%s%s%s%s\n",
1498 nvdimm_name(nvdimm),
1499 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
1500 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
1501 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
1502 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "");
1503
1504 }
1505
1506 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
1507 if (rc)
1508 return rc;
1509
1510 /*
1511 * Now that dimms are successfully registered, and async registration
1512 * is flushed, attempt to enable event notification.
1513 */
1514 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1515 struct kernfs_node *nfit_kernfs;
1516
1517 nvdimm = nfit_mem->nvdimm;
1518 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
1519 if (nfit_kernfs)
1520 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
1521 "flags");
1522 sysfs_put(nfit_kernfs);
1523 if (!nfit_mem->flags_attr)
1524 dev_warn(acpi_desc->dev, "%s: notifications disabled\n",
1525 nvdimm_name(nvdimm));
1526 }
1527
1528 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify,
1529 acpi_desc);
1530 }
1531
1532 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
1533 {
1534 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1535 const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS);
1536 struct acpi_device *adev;
1537 int i;
1538
1539 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
1540 adev = to_acpi_dev(acpi_desc);
1541 if (!adev)
1542 return;
1543
1544 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
1545 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i))
1546 set_bit(i, &nd_desc->cmd_mask);
1547 }
1548
1549 static ssize_t range_index_show(struct device *dev,
1550 struct device_attribute *attr, char *buf)
1551 {
1552 struct nd_region *nd_region = to_nd_region(dev);
1553 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
1554
1555 return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
1556 }
1557 static DEVICE_ATTR_RO(range_index);
1558
1559 static struct attribute *acpi_nfit_region_attributes[] = {
1560 &dev_attr_range_index.attr,
1561 NULL,
1562 };
1563
1564 static struct attribute_group acpi_nfit_region_attribute_group = {
1565 .name = "nfit",
1566 .attrs = acpi_nfit_region_attributes,
1567 };
1568
1569 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
1570 &nd_region_attribute_group,
1571 &nd_mapping_attribute_group,
1572 &nd_device_attribute_group,
1573 &nd_numa_attribute_group,
1574 &acpi_nfit_region_attribute_group,
1575 NULL,
1576 };
1577
1578 /* enough info to uniquely specify an interleave set */
1579 struct nfit_set_info {
1580 struct nfit_set_info_map {
1581 u64 region_offset;
1582 u32 serial_number;
1583 u32 pad;
1584 } mapping[0];
1585 };
1586
1587 static size_t sizeof_nfit_set_info(int num_mappings)
1588 {
1589 return sizeof(struct nfit_set_info)
1590 + num_mappings * sizeof(struct nfit_set_info_map);
1591 }
1592
1593 static int cmp_map(const void *m0, const void *m1)
1594 {
1595 const struct nfit_set_info_map *map0 = m0;
1596 const struct nfit_set_info_map *map1 = m1;
1597
1598 return memcmp(&map0->region_offset, &map1->region_offset,
1599 sizeof(u64));
1600 }
1601
1602 /* Retrieve the nth entry referencing this spa */
1603 static struct acpi_nfit_memory_map *memdev_from_spa(
1604 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
1605 {
1606 struct nfit_memdev *nfit_memdev;
1607
1608 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
1609 if (nfit_memdev->memdev->range_index == range_index)
1610 if (n-- == 0)
1611 return nfit_memdev->memdev;
1612 return NULL;
1613 }
1614
1615 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
1616 struct nd_region_desc *ndr_desc,
1617 struct acpi_nfit_system_address *spa)
1618 {
1619 int i, spa_type = nfit_spa_type(spa);
1620 struct device *dev = acpi_desc->dev;
1621 struct nd_interleave_set *nd_set;
1622 u16 nr = ndr_desc->num_mappings;
1623 struct nfit_set_info *info;
1624
1625 if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE)
1626 /* pass */;
1627 else
1628 return 0;
1629
1630 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
1631 if (!nd_set)
1632 return -ENOMEM;
1633
1634 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
1635 if (!info)
1636 return -ENOMEM;
1637 for (i = 0; i < nr; i++) {
1638 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
1639 struct nfit_set_info_map *map = &info->mapping[i];
1640 struct nvdimm *nvdimm = mapping->nvdimm;
1641 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1642 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
1643 spa->range_index, i);
1644
1645 if (!memdev || !nfit_mem->dcr) {
1646 dev_err(dev, "%s: failed to find DCR\n", __func__);
1647 return -ENODEV;
1648 }
1649
1650 map->region_offset = memdev->region_offset;
1651 map->serial_number = nfit_mem->dcr->serial_number;
1652 }
1653
1654 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
1655 cmp_map, NULL);
1656 nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
1657 ndr_desc->nd_set = nd_set;
1658 devm_kfree(dev, info);
1659
1660 return 0;
1661 }
1662
1663 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
1664 {
1665 struct acpi_nfit_interleave *idt = mmio->idt;
1666 u32 sub_line_offset, line_index, line_offset;
1667 u64 line_no, table_skip_count, table_offset;
1668
1669 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
1670 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
1671 line_offset = idt->line_offset[line_index]
1672 * mmio->line_size;
1673 table_offset = table_skip_count * mmio->table_size;
1674
1675 return mmio->base_offset + line_offset + table_offset + sub_line_offset;
1676 }
1677
1678 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
1679 {
1680 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1681 u64 offset = nfit_blk->stat_offset + mmio->size * bw;
1682 const u32 STATUS_MASK = 0x80000037;
1683
1684 if (mmio->num_lines)
1685 offset = to_interleave_offset(offset, mmio);
1686
1687 return readl(mmio->addr.base + offset) & STATUS_MASK;
1688 }
1689
1690 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
1691 resource_size_t dpa, unsigned int len, unsigned int write)
1692 {
1693 u64 cmd, offset;
1694 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
1695
1696 enum {
1697 BCW_OFFSET_MASK = (1ULL << 48)-1,
1698 BCW_LEN_SHIFT = 48,
1699 BCW_LEN_MASK = (1ULL << 8) - 1,
1700 BCW_CMD_SHIFT = 56,
1701 };
1702
1703 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
1704 len = len >> L1_CACHE_SHIFT;
1705 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
1706 cmd |= ((u64) write) << BCW_CMD_SHIFT;
1707
1708 offset = nfit_blk->cmd_offset + mmio->size * bw;
1709 if (mmio->num_lines)
1710 offset = to_interleave_offset(offset, mmio);
1711
1712 writeq(cmd, mmio->addr.base + offset);
1713 nvdimm_flush(nfit_blk->nd_region);
1714
1715 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
1716 readq(mmio->addr.base + offset);
1717 }
1718
1719 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
1720 resource_size_t dpa, void *iobuf, size_t len, int rw,
1721 unsigned int lane)
1722 {
1723 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1724 unsigned int copied = 0;
1725 u64 base_offset;
1726 int rc;
1727
1728 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
1729 + lane * mmio->size;
1730 write_blk_ctl(nfit_blk, lane, dpa, len, rw);
1731 while (len) {
1732 unsigned int c;
1733 u64 offset;
1734
1735 if (mmio->num_lines) {
1736 u32 line_offset;
1737
1738 offset = to_interleave_offset(base_offset + copied,
1739 mmio);
1740 div_u64_rem(offset, mmio->line_size, &line_offset);
1741 c = min_t(size_t, len, mmio->line_size - line_offset);
1742 } else {
1743 offset = base_offset + nfit_blk->bdw_offset;
1744 c = len;
1745 }
1746
1747 if (rw)
1748 memcpy_to_pmem(mmio->addr.aperture + offset,
1749 iobuf + copied, c);
1750 else {
1751 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
1752 mmio_flush_range((void __force *)
1753 mmio->addr.aperture + offset, c);
1754
1755 memcpy_from_pmem(iobuf + copied,
1756 mmio->addr.aperture + offset, c);
1757 }
1758
1759 copied += c;
1760 len -= c;
1761 }
1762
1763 if (rw)
1764 nvdimm_flush(nfit_blk->nd_region);
1765
1766 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
1767 return rc;
1768 }
1769
1770 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
1771 resource_size_t dpa, void *iobuf, u64 len, int rw)
1772 {
1773 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1774 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1775 struct nd_region *nd_region = nfit_blk->nd_region;
1776 unsigned int lane, copied = 0;
1777 int rc = 0;
1778
1779 lane = nd_region_acquire_lane(nd_region);
1780 while (len) {
1781 u64 c = min(len, mmio->size);
1782
1783 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
1784 iobuf + copied, c, rw, lane);
1785 if (rc)
1786 break;
1787
1788 copied += c;
1789 len -= c;
1790 }
1791 nd_region_release_lane(nd_region, lane);
1792
1793 return rc;
1794 }
1795
1796 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
1797 struct acpi_nfit_interleave *idt, u16 interleave_ways)
1798 {
1799 if (idt) {
1800 mmio->num_lines = idt->line_count;
1801 mmio->line_size = idt->line_size;
1802 if (interleave_ways == 0)
1803 return -ENXIO;
1804 mmio->table_size = mmio->num_lines * interleave_ways
1805 * mmio->line_size;
1806 }
1807
1808 return 0;
1809 }
1810
1811 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
1812 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
1813 {
1814 struct nd_cmd_dimm_flags flags;
1815 int rc;
1816
1817 memset(&flags, 0, sizeof(flags));
1818 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
1819 sizeof(flags), NULL);
1820
1821 if (rc >= 0 && flags.status == 0)
1822 nfit_blk->dimm_flags = flags.flags;
1823 else if (rc == -ENOTTY) {
1824 /* fall back to a conservative default */
1825 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
1826 rc = 0;
1827 } else
1828 rc = -ENXIO;
1829
1830 return rc;
1831 }
1832
1833 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
1834 struct device *dev)
1835 {
1836 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1837 struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1838 struct nfit_blk_mmio *mmio;
1839 struct nfit_blk *nfit_blk;
1840 struct nfit_mem *nfit_mem;
1841 struct nvdimm *nvdimm;
1842 int rc;
1843
1844 nvdimm = nd_blk_region_to_dimm(ndbr);
1845 nfit_mem = nvdimm_provider_data(nvdimm);
1846 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
1847 dev_dbg(dev, "%s: missing%s%s%s\n", __func__,
1848 nfit_mem ? "" : " nfit_mem",
1849 (nfit_mem && nfit_mem->dcr) ? "" : " dcr",
1850 (nfit_mem && nfit_mem->bdw) ? "" : " bdw");
1851 return -ENXIO;
1852 }
1853
1854 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
1855 if (!nfit_blk)
1856 return -ENOMEM;
1857 nd_blk_region_set_provider_data(ndbr, nfit_blk);
1858 nfit_blk->nd_region = to_nd_region(dev);
1859
1860 /* map block aperture memory */
1861 nfit_blk->bdw_offset = nfit_mem->bdw->offset;
1862 mmio = &nfit_blk->mmio[BDW];
1863 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address,
1864 nfit_mem->spa_bdw->length, ARCH_MEMREMAP_PMEM);
1865 if (!mmio->addr.base) {
1866 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__,
1867 nvdimm_name(nvdimm));
1868 return -ENOMEM;
1869 }
1870 mmio->size = nfit_mem->bdw->size;
1871 mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
1872 mmio->idt = nfit_mem->idt_bdw;
1873 mmio->spa = nfit_mem->spa_bdw;
1874 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
1875 nfit_mem->memdev_bdw->interleave_ways);
1876 if (rc) {
1877 dev_dbg(dev, "%s: %s failed to init bdw interleave\n",
1878 __func__, nvdimm_name(nvdimm));
1879 return rc;
1880 }
1881
1882 /* map block control memory */
1883 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
1884 nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
1885 mmio = &nfit_blk->mmio[DCR];
1886 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address,
1887 nfit_mem->spa_dcr->length);
1888 if (!mmio->addr.base) {
1889 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__,
1890 nvdimm_name(nvdimm));
1891 return -ENOMEM;
1892 }
1893 mmio->size = nfit_mem->dcr->window_size;
1894 mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
1895 mmio->idt = nfit_mem->idt_dcr;
1896 mmio->spa = nfit_mem->spa_dcr;
1897 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
1898 nfit_mem->memdev_dcr->interleave_ways);
1899 if (rc) {
1900 dev_dbg(dev, "%s: %s failed to init dcr interleave\n",
1901 __func__, nvdimm_name(nvdimm));
1902 return rc;
1903 }
1904
1905 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
1906 if (rc < 0) {
1907 dev_dbg(dev, "%s: %s failed get DIMM flags\n",
1908 __func__, nvdimm_name(nvdimm));
1909 return rc;
1910 }
1911
1912 if (nvdimm_has_flush(nfit_blk->nd_region) < 0)
1913 dev_warn(dev, "unable to guarantee persistence of writes\n");
1914
1915 if (mmio->line_size == 0)
1916 return 0;
1917
1918 if ((u32) nfit_blk->cmd_offset % mmio->line_size
1919 + 8 > mmio->line_size) {
1920 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
1921 return -ENXIO;
1922 } else if ((u32) nfit_blk->stat_offset % mmio->line_size
1923 + 8 > mmio->line_size) {
1924 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
1925 return -ENXIO;
1926 }
1927
1928 return 0;
1929 }
1930
1931 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
1932 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
1933 {
1934 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1935 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1936 int cmd_rc, rc;
1937
1938 cmd->address = spa->address;
1939 cmd->length = spa->length;
1940 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
1941 sizeof(*cmd), &cmd_rc);
1942 if (rc < 0)
1943 return rc;
1944 return cmd_rc;
1945 }
1946
1947 static int ars_start(struct acpi_nfit_desc *acpi_desc, struct nfit_spa *nfit_spa)
1948 {
1949 int rc;
1950 int cmd_rc;
1951 struct nd_cmd_ars_start ars_start;
1952 struct acpi_nfit_system_address *spa = nfit_spa->spa;
1953 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1954
1955 memset(&ars_start, 0, sizeof(ars_start));
1956 ars_start.address = spa->address;
1957 ars_start.length = spa->length;
1958 if (nfit_spa_type(spa) == NFIT_SPA_PM)
1959 ars_start.type = ND_ARS_PERSISTENT;
1960 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
1961 ars_start.type = ND_ARS_VOLATILE;
1962 else
1963 return -ENOTTY;
1964
1965 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
1966 sizeof(ars_start), &cmd_rc);
1967
1968 if (rc < 0)
1969 return rc;
1970 return cmd_rc;
1971 }
1972
1973 static int ars_continue(struct acpi_nfit_desc *acpi_desc)
1974 {
1975 int rc, cmd_rc;
1976 struct nd_cmd_ars_start ars_start;
1977 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1978 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
1979
1980 memset(&ars_start, 0, sizeof(ars_start));
1981 ars_start.address = ars_status->restart_address;
1982 ars_start.length = ars_status->restart_length;
1983 ars_start.type = ars_status->type;
1984 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
1985 sizeof(ars_start), &cmd_rc);
1986 if (rc < 0)
1987 return rc;
1988 return cmd_rc;
1989 }
1990
1991 static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
1992 {
1993 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
1994 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
1995 int rc, cmd_rc;
1996
1997 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
1998 acpi_desc->ars_status_size, &cmd_rc);
1999 if (rc < 0)
2000 return rc;
2001 return cmd_rc;
2002 }
2003
2004 static int ars_status_process_records(struct nvdimm_bus *nvdimm_bus,
2005 struct nd_cmd_ars_status *ars_status)
2006 {
2007 int rc;
2008 u32 i;
2009
2010 for (i = 0; i < ars_status->num_records; i++) {
2011 rc = nvdimm_bus_add_poison(nvdimm_bus,
2012 ars_status->records[i].err_address,
2013 ars_status->records[i].length);
2014 if (rc)
2015 return rc;
2016 }
2017
2018 return 0;
2019 }
2020
2021 static void acpi_nfit_remove_resource(void *data)
2022 {
2023 struct resource *res = data;
2024
2025 remove_resource(res);
2026 }
2027
2028 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
2029 struct nd_region_desc *ndr_desc)
2030 {
2031 struct resource *res, *nd_res = ndr_desc->res;
2032 int is_pmem, ret;
2033
2034 /* No operation if the region is already registered as PMEM */
2035 is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
2036 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
2037 if (is_pmem == REGION_INTERSECTS)
2038 return 0;
2039
2040 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
2041 if (!res)
2042 return -ENOMEM;
2043
2044 res->name = "Persistent Memory";
2045 res->start = nd_res->start;
2046 res->end = nd_res->end;
2047 res->flags = IORESOURCE_MEM;
2048 res->desc = IORES_DESC_PERSISTENT_MEMORY;
2049
2050 ret = insert_resource(&iomem_resource, res);
2051 if (ret)
2052 return ret;
2053
2054 ret = devm_add_action_or_reset(acpi_desc->dev,
2055 acpi_nfit_remove_resource,
2056 res);
2057 if (ret)
2058 return ret;
2059
2060 return 0;
2061 }
2062
2063 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
2064 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc,
2065 struct acpi_nfit_memory_map *memdev,
2066 struct nfit_spa *nfit_spa)
2067 {
2068 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
2069 memdev->device_handle);
2070 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2071 struct nd_blk_region_desc *ndbr_desc;
2072 struct nfit_mem *nfit_mem;
2073 int blk_valid = 0;
2074
2075 if (!nvdimm) {
2076 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
2077 spa->range_index, memdev->device_handle);
2078 return -ENODEV;
2079 }
2080
2081 mapping->nvdimm = nvdimm;
2082 switch (nfit_spa_type(spa)) {
2083 case NFIT_SPA_PM:
2084 case NFIT_SPA_VOLATILE:
2085 mapping->start = memdev->address;
2086 mapping->size = memdev->region_size;
2087 break;
2088 case NFIT_SPA_DCR:
2089 nfit_mem = nvdimm_provider_data(nvdimm);
2090 if (!nfit_mem || !nfit_mem->bdw) {
2091 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
2092 spa->range_index, nvdimm_name(nvdimm));
2093 } else {
2094 mapping->size = nfit_mem->bdw->capacity;
2095 mapping->start = nfit_mem->bdw->start_address;
2096 ndr_desc->num_lanes = nfit_mem->bdw->windows;
2097 blk_valid = 1;
2098 }
2099
2100 ndr_desc->mapping = mapping;
2101 ndr_desc->num_mappings = blk_valid;
2102 ndbr_desc = to_blk_region_desc(ndr_desc);
2103 ndbr_desc->enable = acpi_nfit_blk_region_enable;
2104 ndbr_desc->do_io = acpi_desc->blk_do_io;
2105 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
2106 ndr_desc);
2107 if (!nfit_spa->nd_region)
2108 return -ENOMEM;
2109 break;
2110 }
2111
2112 return 0;
2113 }
2114
2115 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa)
2116 {
2117 return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2118 nfit_spa_type(spa) == NFIT_SPA_VCD ||
2119 nfit_spa_type(spa) == NFIT_SPA_PDISK ||
2120 nfit_spa_type(spa) == NFIT_SPA_PCD);
2121 }
2122
2123 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
2124 struct nfit_spa *nfit_spa)
2125 {
2126 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS];
2127 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2128 struct nd_blk_region_desc ndbr_desc;
2129 struct nd_region_desc *ndr_desc;
2130 struct nfit_memdev *nfit_memdev;
2131 struct nvdimm_bus *nvdimm_bus;
2132 struct resource res;
2133 int count = 0, rc;
2134
2135 if (nfit_spa->nd_region)
2136 return 0;
2137
2138 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) {
2139 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n",
2140 __func__);
2141 return 0;
2142 }
2143
2144 memset(&res, 0, sizeof(res));
2145 memset(&mappings, 0, sizeof(mappings));
2146 memset(&ndbr_desc, 0, sizeof(ndbr_desc));
2147 res.start = spa->address;
2148 res.end = res.start + spa->length - 1;
2149 ndr_desc = &ndbr_desc.ndr_desc;
2150 ndr_desc->res = &res;
2151 ndr_desc->provider_data = nfit_spa;
2152 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
2153 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID)
2154 ndr_desc->numa_node = acpi_map_pxm_to_online_node(
2155 spa->proximity_domain);
2156 else
2157 ndr_desc->numa_node = NUMA_NO_NODE;
2158
2159 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2160 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
2161 struct nd_mapping_desc *mapping;
2162
2163 if (memdev->range_index != spa->range_index)
2164 continue;
2165 if (count >= ND_MAX_MAPPINGS) {
2166 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
2167 spa->range_index, ND_MAX_MAPPINGS);
2168 return -ENXIO;
2169 }
2170 mapping = &mappings[count++];
2171 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc,
2172 memdev, nfit_spa);
2173 if (rc)
2174 goto out;
2175 }
2176
2177 ndr_desc->mapping = mappings;
2178 ndr_desc->num_mappings = count;
2179 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2180 if (rc)
2181 goto out;
2182
2183 nvdimm_bus = acpi_desc->nvdimm_bus;
2184 if (nfit_spa_type(spa) == NFIT_SPA_PM) {
2185 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
2186 if (rc) {
2187 dev_warn(acpi_desc->dev,
2188 "failed to insert pmem resource to iomem: %d\n",
2189 rc);
2190 goto out;
2191 }
2192
2193 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
2194 ndr_desc);
2195 if (!nfit_spa->nd_region)
2196 rc = -ENOMEM;
2197 } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) {
2198 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
2199 ndr_desc);
2200 if (!nfit_spa->nd_region)
2201 rc = -ENOMEM;
2202 } else if (nfit_spa_is_virtual(spa)) {
2203 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
2204 ndr_desc);
2205 if (!nfit_spa->nd_region)
2206 rc = -ENOMEM;
2207 }
2208
2209 out:
2210 if (rc)
2211 dev_err(acpi_desc->dev, "failed to register spa range %d\n",
2212 nfit_spa->spa->range_index);
2213 return rc;
2214 }
2215
2216 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc,
2217 u32 max_ars)
2218 {
2219 struct device *dev = acpi_desc->dev;
2220 struct nd_cmd_ars_status *ars_status;
2221
2222 if (acpi_desc->ars_status && acpi_desc->ars_status_size >= max_ars) {
2223 memset(acpi_desc->ars_status, 0, acpi_desc->ars_status_size);
2224 return 0;
2225 }
2226
2227 if (acpi_desc->ars_status)
2228 devm_kfree(dev, acpi_desc->ars_status);
2229 acpi_desc->ars_status = NULL;
2230 ars_status = devm_kzalloc(dev, max_ars, GFP_KERNEL);
2231 if (!ars_status)
2232 return -ENOMEM;
2233 acpi_desc->ars_status = ars_status;
2234 acpi_desc->ars_status_size = max_ars;
2235 return 0;
2236 }
2237
2238 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc,
2239 struct nfit_spa *nfit_spa)
2240 {
2241 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2242 int rc;
2243
2244 if (!nfit_spa->max_ars) {
2245 struct nd_cmd_ars_cap ars_cap;
2246
2247 memset(&ars_cap, 0, sizeof(ars_cap));
2248 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
2249 if (rc < 0)
2250 return rc;
2251 nfit_spa->max_ars = ars_cap.max_ars_out;
2252 nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
2253 /* check that the supported scrub types match the spa type */
2254 if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE &&
2255 ((ars_cap.status >> 16) & ND_ARS_VOLATILE) == 0)
2256 return -ENOTTY;
2257 else if (nfit_spa_type(spa) == NFIT_SPA_PM &&
2258 ((ars_cap.status >> 16) & ND_ARS_PERSISTENT) == 0)
2259 return -ENOTTY;
2260 }
2261
2262 if (ars_status_alloc(acpi_desc, nfit_spa->max_ars))
2263 return -ENOMEM;
2264
2265 rc = ars_get_status(acpi_desc);
2266 if (rc < 0 && rc != -ENOSPC)
2267 return rc;
2268
2269 if (ars_status_process_records(acpi_desc->nvdimm_bus,
2270 acpi_desc->ars_status))
2271 return -ENOMEM;
2272
2273 return 0;
2274 }
2275
2276 static void acpi_nfit_async_scrub(struct acpi_nfit_desc *acpi_desc,
2277 struct nfit_spa *nfit_spa)
2278 {
2279 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2280 unsigned int overflow_retry = scrub_overflow_abort;
2281 u64 init_ars_start = 0, init_ars_len = 0;
2282 struct device *dev = acpi_desc->dev;
2283 unsigned int tmo = scrub_timeout;
2284 int rc;
2285
2286 if (!nfit_spa->ars_required || !nfit_spa->nd_region)
2287 return;
2288
2289 rc = ars_start(acpi_desc, nfit_spa);
2290 /*
2291 * If we timed out the initial scan we'll still be busy here,
2292 * and will wait another timeout before giving up permanently.
2293 */
2294 if (rc < 0 && rc != -EBUSY)
2295 return;
2296
2297 do {
2298 u64 ars_start, ars_len;
2299
2300 if (acpi_desc->cancel)
2301 break;
2302 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa);
2303 if (rc == -ENOTTY)
2304 break;
2305 if (rc == -EBUSY && !tmo) {
2306 dev_warn(dev, "range %d ars timeout, aborting\n",
2307 spa->range_index);
2308 break;
2309 }
2310
2311 if (rc == -EBUSY) {
2312 /*
2313 * Note, entries may be appended to the list
2314 * while the lock is dropped, but the workqueue
2315 * being active prevents entries being deleted /
2316 * freed.
2317 */
2318 mutex_unlock(&acpi_desc->init_mutex);
2319 ssleep(1);
2320 tmo--;
2321 mutex_lock(&acpi_desc->init_mutex);
2322 continue;
2323 }
2324
2325 /* we got some results, but there are more pending... */
2326 if (rc == -ENOSPC && overflow_retry--) {
2327 if (!init_ars_len) {
2328 init_ars_len = acpi_desc->ars_status->length;
2329 init_ars_start = acpi_desc->ars_status->address;
2330 }
2331 rc = ars_continue(acpi_desc);
2332 }
2333
2334 if (rc < 0) {
2335 dev_warn(dev, "range %d ars continuation failed\n",
2336 spa->range_index);
2337 break;
2338 }
2339
2340 if (init_ars_len) {
2341 ars_start = init_ars_start;
2342 ars_len = init_ars_len;
2343 } else {
2344 ars_start = acpi_desc->ars_status->address;
2345 ars_len = acpi_desc->ars_status->length;
2346 }
2347 dev_dbg(dev, "spa range: %d ars from %#llx + %#llx complete\n",
2348 spa->range_index, ars_start, ars_len);
2349 /* notify the region about new poison entries */
2350 nvdimm_region_notify(nfit_spa->nd_region,
2351 NVDIMM_REVALIDATE_POISON);
2352 break;
2353 } while (1);
2354 }
2355
2356 static void acpi_nfit_scrub(struct work_struct *work)
2357 {
2358 struct device *dev;
2359 u64 init_scrub_length = 0;
2360 struct nfit_spa *nfit_spa;
2361 u64 init_scrub_address = 0;
2362 bool init_ars_done = false;
2363 struct acpi_nfit_desc *acpi_desc;
2364 unsigned int tmo = scrub_timeout;
2365 unsigned int overflow_retry = scrub_overflow_abort;
2366
2367 acpi_desc = container_of(work, typeof(*acpi_desc), work);
2368 dev = acpi_desc->dev;
2369
2370 /*
2371 * We scrub in 2 phases. The first phase waits for any platform
2372 * firmware initiated scrubs to complete and then we go search for the
2373 * affected spa regions to mark them scanned. In the second phase we
2374 * initiate a directed scrub for every range that was not scrubbed in
2375 * phase 1. If we're called for a 'rescan', we harmlessly pass through
2376 * the first phase, but really only care about running phase 2, where
2377 * regions can be notified of new poison.
2378 */
2379
2380 /* process platform firmware initiated scrubs */
2381 retry:
2382 mutex_lock(&acpi_desc->init_mutex);
2383 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
2384 struct nd_cmd_ars_status *ars_status;
2385 struct acpi_nfit_system_address *spa;
2386 u64 ars_start, ars_len;
2387 int rc;
2388
2389 if (acpi_desc->cancel)
2390 break;
2391
2392 if (nfit_spa->nd_region)
2393 continue;
2394
2395 if (init_ars_done) {
2396 /*
2397 * No need to re-query, we're now just
2398 * reconciling all the ranges covered by the
2399 * initial scrub
2400 */
2401 rc = 0;
2402 } else
2403 rc = acpi_nfit_query_poison(acpi_desc, nfit_spa);
2404
2405 if (rc == -ENOTTY) {
2406 /* no ars capability, just register spa and move on */
2407 acpi_nfit_register_region(acpi_desc, nfit_spa);
2408 continue;
2409 }
2410
2411 if (rc == -EBUSY && !tmo) {
2412 /* fallthrough to directed scrub in phase 2 */
2413 dev_warn(dev, "timeout awaiting ars results, continuing...\n");
2414 break;
2415 } else if (rc == -EBUSY) {
2416 mutex_unlock(&acpi_desc->init_mutex);
2417 ssleep(1);
2418 tmo--;
2419 goto retry;
2420 }
2421
2422 /* we got some results, but there are more pending... */
2423 if (rc == -ENOSPC && overflow_retry--) {
2424 ars_status = acpi_desc->ars_status;
2425 /*
2426 * Record the original scrub range, so that we
2427 * can recall all the ranges impacted by the
2428 * initial scrub.
2429 */
2430 if (!init_scrub_length) {
2431 init_scrub_length = ars_status->length;
2432 init_scrub_address = ars_status->address;
2433 }
2434 rc = ars_continue(acpi_desc);
2435 if (rc == 0) {
2436 mutex_unlock(&acpi_desc->init_mutex);
2437 goto retry;
2438 }
2439 }
2440
2441 if (rc < 0) {
2442 /*
2443 * Initial scrub failed, we'll give it one more
2444 * try below...
2445 */
2446 break;
2447 }
2448
2449 /* We got some final results, record completed ranges */
2450 ars_status = acpi_desc->ars_status;
2451 if (init_scrub_length) {
2452 ars_start = init_scrub_address;
2453 ars_len = ars_start + init_scrub_length;
2454 } else {
2455 ars_start = ars_status->address;
2456 ars_len = ars_status->length;
2457 }
2458 spa = nfit_spa->spa;
2459
2460 if (!init_ars_done) {
2461 init_ars_done = true;
2462 dev_dbg(dev, "init scrub %#llx + %#llx complete\n",
2463 ars_start, ars_len);
2464 }
2465 if (ars_start <= spa->address && ars_start + ars_len
2466 >= spa->address + spa->length)
2467 acpi_nfit_register_region(acpi_desc, nfit_spa);
2468 }
2469
2470 /*
2471 * For all the ranges not covered by an initial scrub we still
2472 * want to see if there are errors, but it's ok to discover them
2473 * asynchronously.
2474 */
2475 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
2476 /*
2477 * Flag all the ranges that still need scrubbing, but
2478 * register them now to make data available.
2479 */
2480 if (!nfit_spa->nd_region) {
2481 nfit_spa->ars_required = 1;
2482 acpi_nfit_register_region(acpi_desc, nfit_spa);
2483 }
2484 }
2485
2486 list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
2487 acpi_nfit_async_scrub(acpi_desc, nfit_spa);
2488 acpi_desc->scrub_count++;
2489 if (acpi_desc->scrub_count_state)
2490 sysfs_notify_dirent(acpi_desc->scrub_count_state);
2491 mutex_unlock(&acpi_desc->init_mutex);
2492 }
2493
2494 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
2495 {
2496 struct nfit_spa *nfit_spa;
2497 int rc;
2498
2499 list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
2500 if (nfit_spa_type(nfit_spa->spa) == NFIT_SPA_DCR) {
2501 /* BLK regions don't need to wait for ars results */
2502 rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
2503 if (rc)
2504 return rc;
2505 }
2506
2507 queue_work(nfit_wq, &acpi_desc->work);
2508 return 0;
2509 }
2510
2511 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
2512 struct nfit_table_prev *prev)
2513 {
2514 struct device *dev = acpi_desc->dev;
2515
2516 if (!list_empty(&prev->spas) ||
2517 !list_empty(&prev->memdevs) ||
2518 !list_empty(&prev->dcrs) ||
2519 !list_empty(&prev->bdws) ||
2520 !list_empty(&prev->idts) ||
2521 !list_empty(&prev->flushes)) {
2522 dev_err(dev, "new nfit deletes entries (unsupported)\n");
2523 return -ENXIO;
2524 }
2525 return 0;
2526 }
2527
2528 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc)
2529 {
2530 struct device *dev = acpi_desc->dev;
2531 struct kernfs_node *nfit;
2532 struct device *bus_dev;
2533
2534 if (!ars_supported(acpi_desc->nvdimm_bus))
2535 return 0;
2536
2537 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
2538 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit");
2539 if (!nfit) {
2540 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n");
2541 return -ENODEV;
2542 }
2543 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub");
2544 sysfs_put(nfit);
2545 if (!acpi_desc->scrub_count_state) {
2546 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n");
2547 return -ENODEV;
2548 }
2549
2550 return 0;
2551 }
2552
2553 static void acpi_nfit_destruct(void *data)
2554 {
2555 struct acpi_nfit_desc *acpi_desc = data;
2556 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
2557
2558 /*
2559 * Destruct under acpi_desc_lock so that nfit_handle_mce does not
2560 * race teardown
2561 */
2562 mutex_lock(&acpi_desc_lock);
2563 acpi_desc->cancel = 1;
2564 /*
2565 * Bounce the nvdimm bus lock to make sure any in-flight
2566 * acpi_nfit_ars_rescan() submissions have had a chance to
2567 * either submit or see ->cancel set.
2568 */
2569 device_lock(bus_dev);
2570 device_unlock(bus_dev);
2571
2572 flush_workqueue(nfit_wq);
2573 if (acpi_desc->scrub_count_state)
2574 sysfs_put(acpi_desc->scrub_count_state);
2575 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
2576 acpi_desc->nvdimm_bus = NULL;
2577 list_del(&acpi_desc->list);
2578 mutex_unlock(&acpi_desc_lock);
2579 }
2580
2581 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
2582 {
2583 struct device *dev = acpi_desc->dev;
2584 struct nfit_table_prev prev;
2585 const void *end;
2586 int rc;
2587
2588 if (!acpi_desc->nvdimm_bus) {
2589 acpi_nfit_init_dsms(acpi_desc);
2590
2591 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev,
2592 &acpi_desc->nd_desc);
2593 if (!acpi_desc->nvdimm_bus)
2594 return -ENOMEM;
2595
2596 rc = devm_add_action_or_reset(dev, acpi_nfit_destruct,
2597 acpi_desc);
2598 if (rc)
2599 return rc;
2600
2601 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc);
2602 if (rc)
2603 return rc;
2604
2605 /* register this acpi_desc for mce notifications */
2606 mutex_lock(&acpi_desc_lock);
2607 list_add_tail(&acpi_desc->list, &acpi_descs);
2608 mutex_unlock(&acpi_desc_lock);
2609 }
2610
2611 mutex_lock(&acpi_desc->init_mutex);
2612
2613 INIT_LIST_HEAD(&prev.spas);
2614 INIT_LIST_HEAD(&prev.memdevs);
2615 INIT_LIST_HEAD(&prev.dcrs);
2616 INIT_LIST_HEAD(&prev.bdws);
2617 INIT_LIST_HEAD(&prev.idts);
2618 INIT_LIST_HEAD(&prev.flushes);
2619
2620 list_cut_position(&prev.spas, &acpi_desc->spas,
2621 acpi_desc->spas.prev);
2622 list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
2623 acpi_desc->memdevs.prev);
2624 list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
2625 acpi_desc->dcrs.prev);
2626 list_cut_position(&prev.bdws, &acpi_desc->bdws,
2627 acpi_desc->bdws.prev);
2628 list_cut_position(&prev.idts, &acpi_desc->idts,
2629 acpi_desc->idts.prev);
2630 list_cut_position(&prev.flushes, &acpi_desc->flushes,
2631 acpi_desc->flushes.prev);
2632
2633 end = data + sz;
2634 while (!IS_ERR_OR_NULL(data))
2635 data = add_table(acpi_desc, &prev, data, end);
2636
2637 if (IS_ERR(data)) {
2638 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__,
2639 PTR_ERR(data));
2640 rc = PTR_ERR(data);
2641 goto out_unlock;
2642 }
2643
2644 rc = acpi_nfit_check_deletions(acpi_desc, &prev);
2645 if (rc)
2646 goto out_unlock;
2647
2648 rc = nfit_mem_init(acpi_desc);
2649 if (rc)
2650 goto out_unlock;
2651
2652 rc = acpi_nfit_register_dimms(acpi_desc);
2653 if (rc)
2654 goto out_unlock;
2655
2656 rc = acpi_nfit_register_regions(acpi_desc);
2657
2658 out_unlock:
2659 mutex_unlock(&acpi_desc->init_mutex);
2660 return rc;
2661 }
2662 EXPORT_SYMBOL_GPL(acpi_nfit_init);
2663
2664 struct acpi_nfit_flush_work {
2665 struct work_struct work;
2666 struct completion cmp;
2667 };
2668
2669 static void flush_probe(struct work_struct *work)
2670 {
2671 struct acpi_nfit_flush_work *flush;
2672
2673 flush = container_of(work, typeof(*flush), work);
2674 complete(&flush->cmp);
2675 }
2676
2677 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
2678 {
2679 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
2680 struct device *dev = acpi_desc->dev;
2681 struct acpi_nfit_flush_work flush;
2682
2683 /* bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
2684 device_lock(dev);
2685 device_unlock(dev);
2686
2687 /*
2688 * Scrub work could take 10s of seconds, userspace may give up so we
2689 * need to be interruptible while waiting.
2690 */
2691 INIT_WORK_ONSTACK(&flush.work, flush_probe);
2692 COMPLETION_INITIALIZER_ONSTACK(flush.cmp);
2693 queue_work(nfit_wq, &flush.work);
2694 return wait_for_completion_interruptible(&flush.cmp);
2695 }
2696
2697 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
2698 struct nvdimm *nvdimm, unsigned int cmd)
2699 {
2700 struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
2701
2702 if (nvdimm)
2703 return 0;
2704 if (cmd != ND_CMD_ARS_START)
2705 return 0;
2706
2707 /*
2708 * The kernel and userspace may race to initiate a scrub, but
2709 * the scrub thread is prepared to lose that initial race. It
2710 * just needs guarantees that any ars it initiates are not
2711 * interrupted by any intervening start reqeusts from userspace.
2712 */
2713 if (work_busy(&acpi_desc->work))
2714 return -EBUSY;
2715
2716 return 0;
2717 }
2718
2719 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc)
2720 {
2721 struct device *dev = acpi_desc->dev;
2722 struct nfit_spa *nfit_spa;
2723
2724 if (work_busy(&acpi_desc->work))
2725 return -EBUSY;
2726
2727 if (acpi_desc->cancel)
2728 return 0;
2729
2730 mutex_lock(&acpi_desc->init_mutex);
2731 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
2732 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2733
2734 if (nfit_spa_type(spa) != NFIT_SPA_PM)
2735 continue;
2736
2737 nfit_spa->ars_required = 1;
2738 }
2739 queue_work(nfit_wq, &acpi_desc->work);
2740 dev_dbg(dev, "%s: ars_scan triggered\n", __func__);
2741 mutex_unlock(&acpi_desc->init_mutex);
2742
2743 return 0;
2744 }
2745
2746 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
2747 {
2748 struct nvdimm_bus_descriptor *nd_desc;
2749
2750 dev_set_drvdata(dev, acpi_desc);
2751 acpi_desc->dev = dev;
2752 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
2753 nd_desc = &acpi_desc->nd_desc;
2754 nd_desc->provider_name = "ACPI.NFIT";
2755 nd_desc->module = THIS_MODULE;
2756 nd_desc->ndctl = acpi_nfit_ctl;
2757 nd_desc->flush_probe = acpi_nfit_flush_probe;
2758 nd_desc->clear_to_send = acpi_nfit_clear_to_send;
2759 nd_desc->attr_groups = acpi_nfit_attribute_groups;
2760
2761 INIT_LIST_HEAD(&acpi_desc->spas);
2762 INIT_LIST_HEAD(&acpi_desc->dcrs);
2763 INIT_LIST_HEAD(&acpi_desc->bdws);
2764 INIT_LIST_HEAD(&acpi_desc->idts);
2765 INIT_LIST_HEAD(&acpi_desc->flushes);
2766 INIT_LIST_HEAD(&acpi_desc->memdevs);
2767 INIT_LIST_HEAD(&acpi_desc->dimms);
2768 INIT_LIST_HEAD(&acpi_desc->list);
2769 mutex_init(&acpi_desc->init_mutex);
2770 INIT_WORK(&acpi_desc->work, acpi_nfit_scrub);
2771 }
2772 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
2773
2774 static int acpi_nfit_add(struct acpi_device *adev)
2775 {
2776 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
2777 struct acpi_nfit_desc *acpi_desc;
2778 struct device *dev = &adev->dev;
2779 struct acpi_table_header *tbl;
2780 acpi_status status = AE_OK;
2781 acpi_size sz;
2782 int rc = 0;
2783
2784 status = acpi_get_table_with_size(ACPI_SIG_NFIT, 0, &tbl, &sz);
2785 if (ACPI_FAILURE(status)) {
2786 /* This is ok, we could have an nvdimm hotplugged later */
2787 dev_dbg(dev, "failed to find NFIT at startup\n");
2788 return 0;
2789 }
2790
2791 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
2792 if (!acpi_desc)
2793 return -ENOMEM;
2794 acpi_nfit_desc_init(acpi_desc, &adev->dev);
2795
2796 /* Save the acpi header for exporting the revision via sysfs */
2797 acpi_desc->acpi_header = *tbl;
2798
2799 /* Evaluate _FIT and override with that if present */
2800 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
2801 if (ACPI_SUCCESS(status) && buf.length > 0) {
2802 union acpi_object *obj = buf.pointer;
2803
2804 if (obj->type == ACPI_TYPE_BUFFER)
2805 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
2806 obj->buffer.length);
2807 else
2808 dev_dbg(dev, "%s invalid type %d, ignoring _FIT\n",
2809 __func__, (int) obj->type);
2810 kfree(buf.pointer);
2811 } else
2812 /* skip over the lead-in header table */
2813 rc = acpi_nfit_init(acpi_desc, (void *) tbl
2814 + sizeof(struct acpi_table_nfit),
2815 sz - sizeof(struct acpi_table_nfit));
2816 return rc;
2817 }
2818
2819 static int acpi_nfit_remove(struct acpi_device *adev)
2820 {
2821 /* see acpi_nfit_destruct */
2822 return 0;
2823 }
2824
2825 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event)
2826 {
2827 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
2828 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
2829 union acpi_object *obj;
2830 acpi_status status;
2831 int ret;
2832
2833 dev_dbg(dev, "%s: event: %d\n", __func__, event);
2834
2835 if (event != NFIT_NOTIFY_UPDATE)
2836 return;
2837
2838 if (!dev->driver) {
2839 /* dev->driver may be null if we're being removed */
2840 dev_dbg(dev, "%s: no driver found for dev\n", __func__);
2841 return;
2842 }
2843
2844 if (!acpi_desc) {
2845 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
2846 if (!acpi_desc)
2847 return;
2848 acpi_nfit_desc_init(acpi_desc, dev);
2849 } else {
2850 /*
2851 * Finish previous registration before considering new
2852 * regions.
2853 */
2854 flush_workqueue(nfit_wq);
2855 }
2856
2857 /* Evaluate _FIT */
2858 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf);
2859 if (ACPI_FAILURE(status)) {
2860 dev_err(dev, "failed to evaluate _FIT\n");
2861 return;
2862 }
2863
2864 obj = buf.pointer;
2865 if (obj->type == ACPI_TYPE_BUFFER) {
2866 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
2867 obj->buffer.length);
2868 if (ret)
2869 dev_err(dev, "failed to merge updated NFIT\n");
2870 } else
2871 dev_err(dev, "Invalid _FIT\n");
2872 kfree(buf.pointer);
2873 }
2874 EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
2875
2876 static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
2877 {
2878 device_lock(&adev->dev);
2879 __acpi_nfit_notify(&adev->dev, adev->handle, event);
2880 device_unlock(&adev->dev);
2881 }
2882
2883 static const struct acpi_device_id acpi_nfit_ids[] = {
2884 { "ACPI0012", 0 },
2885 { "", 0 },
2886 };
2887 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
2888
2889 static struct acpi_driver acpi_nfit_driver = {
2890 .name = KBUILD_MODNAME,
2891 .ids = acpi_nfit_ids,
2892 .ops = {
2893 .add = acpi_nfit_add,
2894 .remove = acpi_nfit_remove,
2895 .notify = acpi_nfit_notify,
2896 },
2897 };
2898
2899 static __init int nfit_init(void)
2900 {
2901 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
2902 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
2903 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
2904 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
2905 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
2906 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
2907 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
2908
2909 acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]);
2910 acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]);
2911 acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]);
2912 acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]);
2913 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]);
2914 acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]);
2915 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]);
2916 acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]);
2917 acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]);
2918 acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]);
2919 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE1, nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
2920 acpi_str_to_uuid(UUID_NFIT_DIMM_N_HPE2, nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
2921 acpi_str_to_uuid(UUID_NFIT_DIMM_N_MSFT, nfit_uuid[NFIT_DEV_DIMM_N_MSFT]);
2922
2923 nfit_wq = create_singlethread_workqueue("nfit");
2924 if (!nfit_wq)
2925 return -ENOMEM;
2926
2927 nfit_mce_register();
2928
2929 return acpi_bus_register_driver(&acpi_nfit_driver);
2930 }
2931
2932 static __exit void nfit_exit(void)
2933 {
2934 nfit_mce_unregister();
2935 acpi_bus_unregister_driver(&acpi_nfit_driver);
2936 destroy_workqueue(nfit_wq);
2937 WARN_ON(!list_empty(&acpi_descs));
2938 }
2939
2940 module_init(nfit_init);
2941 module_exit(nfit_exit);
2942 MODULE_LICENSE("GPL v2");
2943 MODULE_AUTHOR("Intel Corporation");
Linux with added FPC-III support