?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / acpi / nfit / core.c
bloba3320f93616dec56cb7c6015962525454d4b8a9c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 */
5 #include <linux/list_sort.h>
6 #include <linux/libnvdimm.h>
7 #include <linux/module.h>
8 #include <linux/mutex.h>
9 #include <linux/ndctl.h>
10 #include <linux/sysfs.h>
11 #include <linux/delay.h>
12 #include <linux/list.h>
13 #include <linux/acpi.h>
14 #include <linux/sort.h>
15 #include <linux/io.h>
16 #include <linux/nd.h>
17 #include <asm/cacheflush.h>
18 #include <acpi/nfit.h>
19 #include "intel.h"
20 #include "nfit.h"
21
22 /*
23 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
24 * irrelevant.
25 */
26 #include <linux/io-64-nonatomic-hi-lo.h>
27
28 static bool force_enable_dimms;
29 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
30 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
31
32 static bool disable_vendor_specific;
33 module_param(disable_vendor_specific, bool, S_IRUGO);
34 MODULE_PARM_DESC(disable_vendor_specific,
35 "Limit commands to the publicly specified set");
36
37 static unsigned long override_dsm_mask;
38 module_param(override_dsm_mask, ulong, S_IRUGO);
39 MODULE_PARM_DESC(override_dsm_mask, "Bitmask of allowed NVDIMM DSM functions");
40
41 static int default_dsm_family = -1;
42 module_param(default_dsm_family, int, S_IRUGO);
43 MODULE_PARM_DESC(default_dsm_family,
44 "Try this DSM type first when identifying NVDIMM family");
45
46 static bool no_init_ars;
47 module_param(no_init_ars, bool, 0644);
48 MODULE_PARM_DESC(no_init_ars, "Skip ARS run at nfit init time");
49
50 static bool force_labels;
51 module_param(force_labels, bool, 0444);
52 MODULE_PARM_DESC(force_labels, "Opt-in to labels despite missing methods");
53
54 LIST_HEAD(acpi_descs);
55 DEFINE_MUTEX(acpi_desc_lock);
56
57 static struct workqueue_struct *nfit_wq;
58
59 struct nfit_table_prev {
60 struct list_head spas;
61 struct list_head memdevs;
62 struct list_head dcrs;
63 struct list_head bdws;
64 struct list_head idts;
65 struct list_head flushes;
66 };
67
68 static guid_t nfit_uuid[NFIT_UUID_MAX];
69
70 const guid_t *to_nfit_uuid(enum nfit_uuids id)
71 {
72 return &nfit_uuid[id];
73 }
74 EXPORT_SYMBOL(to_nfit_uuid);
75
76 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
77 {
78 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
79
80 /*
81 * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
82 * acpi_device.
83 */
84 if (!nd_desc->provider_name
85 || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
86 return NULL;
87
88 return to_acpi_device(acpi_desc->dev);
89 }
90
91 static int xlat_bus_status(void *buf, unsigned int cmd, u32 status)
92 {
93 struct nd_cmd_clear_error *clear_err;
94 struct nd_cmd_ars_status *ars_status;
95 u16 flags;
96
97 switch (cmd) {
98 case ND_CMD_ARS_CAP:
99 if ((status & 0xffff) == NFIT_ARS_CAP_NONE)
100 return -ENOTTY;
101
102 /* Command failed */
103 if (status & 0xffff)
104 return -EIO;
105
106 /* No supported scan types for this range */
107 flags = ND_ARS_PERSISTENT | ND_ARS_VOLATILE;
108 if ((status >> 16 & flags) == 0)
109 return -ENOTTY;
110 return 0;
111 case ND_CMD_ARS_START:
112 /* ARS is in progress */
113 if ((status & 0xffff) == NFIT_ARS_START_BUSY)
114 return -EBUSY;
115
116 /* Command failed */
117 if (status & 0xffff)
118 return -EIO;
119 return 0;
120 case ND_CMD_ARS_STATUS:
121 ars_status = buf;
122 /* Command failed */
123 if (status & 0xffff)
124 return -EIO;
125 /* Check extended status (Upper two bytes) */
126 if (status == NFIT_ARS_STATUS_DONE)
127 return 0;
128
129 /* ARS is in progress */
130 if (status == NFIT_ARS_STATUS_BUSY)
131 return -EBUSY;
132
133 /* No ARS performed for the current boot */
134 if (status == NFIT_ARS_STATUS_NONE)
135 return -EAGAIN;
136
137 /*
138 * ARS interrupted, either we overflowed or some other
139 * agent wants the scan to stop. If we didn't overflow
140 * then just continue with the returned results.
141 */
142 if (status == NFIT_ARS_STATUS_INTR) {
143 if (ars_status->out_length >= 40 && (ars_status->flags
144 & NFIT_ARS_F_OVERFLOW))
145 return -ENOSPC;
146 return 0;
147 }
148
149 /* Unknown status */
150 if (status >> 16)
151 return -EIO;
152 return 0;
153 case ND_CMD_CLEAR_ERROR:
154 clear_err = buf;
155 if (status & 0xffff)
156 return -EIO;
157 if (!clear_err->cleared)
158 return -EIO;
159 if (clear_err->length > clear_err->cleared)
160 return clear_err->cleared;
161 return 0;
162 default:
163 break;
164 }
165
166 /* all other non-zero status results in an error */
167 if (status)
168 return -EIO;
169 return 0;
170 }
171
172 #define ACPI_LABELS_LOCKED 3
173
174 static int xlat_nvdimm_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
175 u32 status)
176 {
177 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
178
179 switch (cmd) {
180 case ND_CMD_GET_CONFIG_SIZE:
181 /*
182 * In the _LSI, _LSR, _LSW case the locked status is
183 * communicated via the read/write commands
184 */
185 if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
186 break;
187
188 if (status >> 16 & ND_CONFIG_LOCKED)
189 return -EACCES;
190 break;
191 case ND_CMD_GET_CONFIG_DATA:
192 if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
193 && status == ACPI_LABELS_LOCKED)
194 return -EACCES;
195 break;
196 case ND_CMD_SET_CONFIG_DATA:
197 if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags)
198 && status == ACPI_LABELS_LOCKED)
199 return -EACCES;
200 break;
201 default:
202 break;
203 }
204
205 /* all other non-zero status results in an error */
206 if (status)
207 return -EIO;
208 return 0;
209 }
210
211 static int xlat_status(struct nvdimm *nvdimm, void *buf, unsigned int cmd,
212 u32 status)
213 {
214 if (!nvdimm)
215 return xlat_bus_status(buf, cmd, status);
216 return xlat_nvdimm_status(nvdimm, buf, cmd, status);
217 }
218
219 /* convert _LS{I,R} packages to the buffer object acpi_nfit_ctl expects */
220 static union acpi_object *pkg_to_buf(union acpi_object *pkg)
221 {
222 int i;
223 void *dst;
224 size_t size = 0;
225 union acpi_object *buf = NULL;
226
227 if (pkg->type != ACPI_TYPE_PACKAGE) {
228 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
229 pkg->type);
230 goto err;
231 }
232
233 for (i = 0; i < pkg->package.count; i++) {
234 union acpi_object *obj = &pkg->package.elements[i];
235
236 if (obj->type == ACPI_TYPE_INTEGER)
237 size += 4;
238 else if (obj->type == ACPI_TYPE_BUFFER)
239 size += obj->buffer.length;
240 else {
241 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
242 obj->type);
243 goto err;
244 }
245 }
246
247 buf = ACPI_ALLOCATE(sizeof(*buf) + size);
248 if (!buf)
249 goto err;
250
251 dst = buf + 1;
252 buf->type = ACPI_TYPE_BUFFER;
253 buf->buffer.length = size;
254 buf->buffer.pointer = dst;
255 for (i = 0; i < pkg->package.count; i++) {
256 union acpi_object *obj = &pkg->package.elements[i];
257
258 if (obj->type == ACPI_TYPE_INTEGER) {
259 memcpy(dst, &obj->integer.value, 4);
260 dst += 4;
261 } else if (obj->type == ACPI_TYPE_BUFFER) {
262 memcpy(dst, obj->buffer.pointer, obj->buffer.length);
263 dst += obj->buffer.length;
264 }
265 }
266 err:
267 ACPI_FREE(pkg);
268 return buf;
269 }
270
271 static union acpi_object *int_to_buf(union acpi_object *integer)
272 {
273 union acpi_object *buf = ACPI_ALLOCATE(sizeof(*buf) + 4);
274 void *dst = NULL;
275
276 if (!buf)
277 goto err;
278
279 if (integer->type != ACPI_TYPE_INTEGER) {
280 WARN_ONCE(1, "BIOS bug, unexpected element type: %d\n",
281 integer->type);
282 goto err;
283 }
284
285 dst = buf + 1;
286 buf->type = ACPI_TYPE_BUFFER;
287 buf->buffer.length = 4;
288 buf->buffer.pointer = dst;
289 memcpy(dst, &integer->integer.value, 4);
290 err:
291 ACPI_FREE(integer);
292 return buf;
293 }
294
295 static union acpi_object *acpi_label_write(acpi_handle handle, u32 offset,
296 u32 len, void *data)
297 {
298 acpi_status rc;
299 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
300 struct acpi_object_list input = {
301 .count = 3,
302 .pointer = (union acpi_object []) {
303 [0] = {
304 .integer.type = ACPI_TYPE_INTEGER,
305 .integer.value = offset,
306 },
307 [1] = {
308 .integer.type = ACPI_TYPE_INTEGER,
309 .integer.value = len,
310 },
311 [2] = {
312 .buffer.type = ACPI_TYPE_BUFFER,
313 .buffer.pointer = data,
314 .buffer.length = len,
315 },
316 },
317 };
318
319 rc = acpi_evaluate_object(handle, "_LSW", &input, &buf);
320 if (ACPI_FAILURE(rc))
321 return NULL;
322 return int_to_buf(buf.pointer);
323 }
324
325 static union acpi_object *acpi_label_read(acpi_handle handle, u32 offset,
326 u32 len)
327 {
328 acpi_status rc;
329 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
330 struct acpi_object_list input = {
331 .count = 2,
332 .pointer = (union acpi_object []) {
333 [0] = {
334 .integer.type = ACPI_TYPE_INTEGER,
335 .integer.value = offset,
336 },
337 [1] = {
338 .integer.type = ACPI_TYPE_INTEGER,
339 .integer.value = len,
340 },
341 },
342 };
343
344 rc = acpi_evaluate_object(handle, "_LSR", &input, &buf);
345 if (ACPI_FAILURE(rc))
346 return NULL;
347 return pkg_to_buf(buf.pointer);
348 }
349
350 static union acpi_object *acpi_label_info(acpi_handle handle)
351 {
352 acpi_status rc;
353 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
354
355 rc = acpi_evaluate_object(handle, "_LSI", NULL, &buf);
356 if (ACPI_FAILURE(rc))
357 return NULL;
358 return pkg_to_buf(buf.pointer);
359 }
360
361 static u8 nfit_dsm_revid(unsigned family, unsigned func)
362 {
363 static const u8 revid_table[NVDIMM_FAMILY_MAX+1][32] = {
364 [NVDIMM_FAMILY_INTEL] = {
365 [NVDIMM_INTEL_GET_MODES] = 2,
366 [NVDIMM_INTEL_GET_FWINFO] = 2,
367 [NVDIMM_INTEL_START_FWUPDATE] = 2,
368 [NVDIMM_INTEL_SEND_FWUPDATE] = 2,
369 [NVDIMM_INTEL_FINISH_FWUPDATE] = 2,
370 [NVDIMM_INTEL_QUERY_FWUPDATE] = 2,
371 [NVDIMM_INTEL_SET_THRESHOLD] = 2,
372 [NVDIMM_INTEL_INJECT_ERROR] = 2,
373 [NVDIMM_INTEL_GET_SECURITY_STATE] = 2,
374 [NVDIMM_INTEL_SET_PASSPHRASE] = 2,
375 [NVDIMM_INTEL_DISABLE_PASSPHRASE] = 2,
376 [NVDIMM_INTEL_UNLOCK_UNIT] = 2,
377 [NVDIMM_INTEL_FREEZE_LOCK] = 2,
378 [NVDIMM_INTEL_SECURE_ERASE] = 2,
379 [NVDIMM_INTEL_OVERWRITE] = 2,
380 [NVDIMM_INTEL_QUERY_OVERWRITE] = 2,
381 [NVDIMM_INTEL_SET_MASTER_PASSPHRASE] = 2,
382 [NVDIMM_INTEL_MASTER_SECURE_ERASE] = 2,
383 },
384 };
385 u8 id;
386
387 if (family > NVDIMM_FAMILY_MAX)
388 return 0;
389 if (func > 31)
390 return 0;
391 id = revid_table[family][func];
392 if (id == 0)
393 return 1; /* default */
394 return id;
395 }
396
397 static bool payload_dumpable(struct nvdimm *nvdimm, unsigned int func)
398 {
399 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
400
401 if (nfit_mem && nfit_mem->family == NVDIMM_FAMILY_INTEL
402 && func >= NVDIMM_INTEL_GET_SECURITY_STATE
403 && func <= NVDIMM_INTEL_MASTER_SECURE_ERASE)
404 return IS_ENABLED(CONFIG_NFIT_SECURITY_DEBUG);
405 return true;
406 }
407
408 static int cmd_to_func(struct nfit_mem *nfit_mem, unsigned int cmd,
409 struct nd_cmd_pkg *call_pkg)
410 {
411 if (call_pkg) {
412 int i;
413
414 if (nfit_mem && nfit_mem->family != call_pkg->nd_family)
415 return -ENOTTY;
416
417 for (i = 0; i < ARRAY_SIZE(call_pkg->nd_reserved2); i++)
418 if (call_pkg->nd_reserved2[i])
419 return -EINVAL;
420 return call_pkg->nd_command;
421 }
422
423 /* In the !call_pkg case, bus commands == bus functions */
424 if (!nfit_mem)
425 return cmd;
426
427 /* Linux ND commands == NVDIMM_FAMILY_INTEL function numbers */
428 if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
429 return cmd;
430
431 /*
432 * Force function number validation to fail since 0 is never
433 * published as a valid function in dsm_mask.
434 */
435 return 0;
436 }
437
438 int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc, struct nvdimm *nvdimm,
439 unsigned int cmd, void *buf, unsigned int buf_len, int *cmd_rc)
440 {
441 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
442 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
443 union acpi_object in_obj, in_buf, *out_obj;
444 const struct nd_cmd_desc *desc = NULL;
445 struct device *dev = acpi_desc->dev;
446 struct nd_cmd_pkg *call_pkg = NULL;
447 const char *cmd_name, *dimm_name;
448 unsigned long cmd_mask, dsm_mask;
449 u32 offset, fw_status = 0;
450 acpi_handle handle;
451 const guid_t *guid;
452 int func, rc, i;
453
454 if (cmd_rc)
455 *cmd_rc = -EINVAL;
456
457 if (cmd == ND_CMD_CALL)
458 call_pkg = buf;
459 func = cmd_to_func(nfit_mem, cmd, call_pkg);
460 if (func < 0)
461 return func;
462
463 if (nvdimm) {
464 struct acpi_device *adev = nfit_mem->adev;
465
466 if (!adev)
467 return -ENOTTY;
468
469 dimm_name = nvdimm_name(nvdimm);
470 cmd_name = nvdimm_cmd_name(cmd);
471 cmd_mask = nvdimm_cmd_mask(nvdimm);
472 dsm_mask = nfit_mem->dsm_mask;
473 desc = nd_cmd_dimm_desc(cmd);
474 guid = to_nfit_uuid(nfit_mem->family);
475 handle = adev->handle;
476 } else {
477 struct acpi_device *adev = to_acpi_dev(acpi_desc);
478
479 cmd_name = nvdimm_bus_cmd_name(cmd);
480 cmd_mask = nd_desc->cmd_mask;
481 dsm_mask = nd_desc->bus_dsm_mask;
482 desc = nd_cmd_bus_desc(cmd);
483 guid = to_nfit_uuid(NFIT_DEV_BUS);
484 handle = adev->handle;
485 dimm_name = "bus";
486 }
487
488 if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
489 return -ENOTTY;
490
491 /*
492 * Check for a valid command. For ND_CMD_CALL, we also have to
493 * make sure that the DSM function is supported.
494 */
495 if (cmd == ND_CMD_CALL && !test_bit(func, &dsm_mask))
496 return -ENOTTY;
497 else if (!test_bit(cmd, &cmd_mask))
498 return -ENOTTY;
499
500 in_obj.type = ACPI_TYPE_PACKAGE;
501 in_obj.package.count = 1;
502 in_obj.package.elements = &in_buf;
503 in_buf.type = ACPI_TYPE_BUFFER;
504 in_buf.buffer.pointer = buf;
505 in_buf.buffer.length = 0;
506
507 /* libnvdimm has already validated the input envelope */
508 for (i = 0; i < desc->in_num; i++)
509 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
510 i, buf);
511
512 if (call_pkg) {
513 /* skip over package wrapper */
514 in_buf.buffer.pointer = (void *) &call_pkg->nd_payload;
515 in_buf.buffer.length = call_pkg->nd_size_in;
516 }
517
518 dev_dbg(dev, "%s cmd: %d: func: %d input length: %d\n",
519 dimm_name, cmd, func, in_buf.buffer.length);
520 if (payload_dumpable(nvdimm, func))
521 print_hex_dump_debug("nvdimm in ", DUMP_PREFIX_OFFSET, 4, 4,
522 in_buf.buffer.pointer,
523 min_t(u32, 256, in_buf.buffer.length), true);
524
525 /* call the BIOS, prefer the named methods over _DSM if available */
526 if (nvdimm && cmd == ND_CMD_GET_CONFIG_SIZE
527 && test_bit(NFIT_MEM_LSR, &nfit_mem->flags))
528 out_obj = acpi_label_info(handle);
529 else if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA
530 && test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
531 struct nd_cmd_get_config_data_hdr *p = buf;
532
533 out_obj = acpi_label_read(handle, p->in_offset, p->in_length);
534 } else if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA
535 && test_bit(NFIT_MEM_LSW, &nfit_mem->flags)) {
536 struct nd_cmd_set_config_hdr *p = buf;
537
538 out_obj = acpi_label_write(handle, p->in_offset, p->in_length,
539 p->in_buf);
540 } else {
541 u8 revid;
542
543 if (nvdimm)
544 revid = nfit_dsm_revid(nfit_mem->family, func);
545 else
546 revid = 1;
547 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
548 }
549
550 if (!out_obj) {
551 dev_dbg(dev, "%s _DSM failed cmd: %s\n", dimm_name, cmd_name);
552 return -EINVAL;
553 }
554
555 if (out_obj->type != ACPI_TYPE_BUFFER) {
556 dev_dbg(dev, "%s unexpected output object type cmd: %s type: %d\n",
557 dimm_name, cmd_name, out_obj->type);
558 rc = -EINVAL;
559 goto out;
560 }
561
562 dev_dbg(dev, "%s cmd: %s output length: %d\n", dimm_name,
563 cmd_name, out_obj->buffer.length);
564 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4, 4,
565 out_obj->buffer.pointer,
566 min_t(u32, 128, out_obj->buffer.length), true);
567
568 if (call_pkg) {
569 call_pkg->nd_fw_size = out_obj->buffer.length;
570 memcpy(call_pkg->nd_payload + call_pkg->nd_size_in,
571 out_obj->buffer.pointer,
572 min(call_pkg->nd_fw_size, call_pkg->nd_size_out));
573
574 ACPI_FREE(out_obj);
575 /*
576 * Need to support FW function w/o known size in advance.
577 * Caller can determine required size based upon nd_fw_size.
578 * If we return an error (like elsewhere) then caller wouldn't
579 * be able to rely upon data returned to make calculation.
580 */
581 if (cmd_rc)
582 *cmd_rc = 0;
583 return 0;
584 }
585
586 for (i = 0, offset = 0; i < desc->out_num; i++) {
587 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
588 (u32 *) out_obj->buffer.pointer,
589 out_obj->buffer.length - offset);
590
591 if (offset + out_size > out_obj->buffer.length) {
592 dev_dbg(dev, "%s output object underflow cmd: %s field: %d\n",
593 dimm_name, cmd_name, i);
594 break;
595 }
596
597 if (in_buf.buffer.length + offset + out_size > buf_len) {
598 dev_dbg(dev, "%s output overrun cmd: %s field: %d\n",
599 dimm_name, cmd_name, i);
600 rc = -ENXIO;
601 goto out;
602 }
603 memcpy(buf + in_buf.buffer.length + offset,
604 out_obj->buffer.pointer + offset, out_size);
605 offset += out_size;
606 }
607
608 /*
609 * Set fw_status for all the commands with a known format to be
610 * later interpreted by xlat_status().
611 */
612 if (i >= 1 && ((!nvdimm && cmd >= ND_CMD_ARS_CAP
613 && cmd <= ND_CMD_CLEAR_ERROR)
614 || (nvdimm && cmd >= ND_CMD_SMART
615 && cmd <= ND_CMD_VENDOR)))
616 fw_status = *(u32 *) out_obj->buffer.pointer;
617
618 if (offset + in_buf.buffer.length < buf_len) {
619 if (i >= 1) {
620 /*
621 * status valid, return the number of bytes left
622 * unfilled in the output buffer
623 */
624 rc = buf_len - offset - in_buf.buffer.length;
625 if (cmd_rc)
626 *cmd_rc = xlat_status(nvdimm, buf, cmd,
627 fw_status);
628 } else {
629 dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
630 __func__, dimm_name, cmd_name, buf_len,
631 offset);
632 rc = -ENXIO;
633 }
634 } else {
635 rc = 0;
636 if (cmd_rc)
637 *cmd_rc = xlat_status(nvdimm, buf, cmd, fw_status);
638 }
639
640 out:
641 ACPI_FREE(out_obj);
642
643 return rc;
644 }
645 EXPORT_SYMBOL_GPL(acpi_nfit_ctl);
646
647 static const char *spa_type_name(u16 type)
648 {
649 static const char *to_name[] = {
650 [NFIT_SPA_VOLATILE] = "volatile",
651 [NFIT_SPA_PM] = "pmem",
652 [NFIT_SPA_DCR] = "dimm-control-region",
653 [NFIT_SPA_BDW] = "block-data-window",
654 [NFIT_SPA_VDISK] = "volatile-disk",
655 [NFIT_SPA_VCD] = "volatile-cd",
656 [NFIT_SPA_PDISK] = "persistent-disk",
657 [NFIT_SPA_PCD] = "persistent-cd",
658
659 };
660
661 if (type > NFIT_SPA_PCD)
662 return "unknown";
663
664 return to_name[type];
665 }
666
667 int nfit_spa_type(struct acpi_nfit_system_address *spa)
668 {
669 int i;
670
671 for (i = 0; i < NFIT_UUID_MAX; i++)
672 if (guid_equal(to_nfit_uuid(i), (guid_t *)&spa->range_guid))
673 return i;
674 return -1;
675 }
676
677 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
678 struct nfit_table_prev *prev,
679 struct acpi_nfit_system_address *spa)
680 {
681 struct device *dev = acpi_desc->dev;
682 struct nfit_spa *nfit_spa;
683
684 if (spa->header.length != sizeof(*spa))
685 return false;
686
687 list_for_each_entry(nfit_spa, &prev->spas, list) {
688 if (memcmp(nfit_spa->spa, spa, sizeof(*spa)) == 0) {
689 list_move_tail(&nfit_spa->list, &acpi_desc->spas);
690 return true;
691 }
692 }
693
694 nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa) + sizeof(*spa),
695 GFP_KERNEL);
696 if (!nfit_spa)
697 return false;
698 INIT_LIST_HEAD(&nfit_spa->list);
699 memcpy(nfit_spa->spa, spa, sizeof(*spa));
700 list_add_tail(&nfit_spa->list, &acpi_desc->spas);
701 dev_dbg(dev, "spa index: %d type: %s\n",
702 spa->range_index,
703 spa_type_name(nfit_spa_type(spa)));
704 return true;
705 }
706
707 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
708 struct nfit_table_prev *prev,
709 struct acpi_nfit_memory_map *memdev)
710 {
711 struct device *dev = acpi_desc->dev;
712 struct nfit_memdev *nfit_memdev;
713
714 if (memdev->header.length != sizeof(*memdev))
715 return false;
716
717 list_for_each_entry(nfit_memdev, &prev->memdevs, list)
718 if (memcmp(nfit_memdev->memdev, memdev, sizeof(*memdev)) == 0) {
719 list_move_tail(&nfit_memdev->list, &acpi_desc->memdevs);
720 return true;
721 }
722
723 nfit_memdev = devm_kzalloc(dev, sizeof(*nfit_memdev) + sizeof(*memdev),
724 GFP_KERNEL);
725 if (!nfit_memdev)
726 return false;
727 INIT_LIST_HEAD(&nfit_memdev->list);
728 memcpy(nfit_memdev->memdev, memdev, sizeof(*memdev));
729 list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
730 dev_dbg(dev, "memdev handle: %#x spa: %d dcr: %d flags: %#x\n",
731 memdev->device_handle, memdev->range_index,
732 memdev->region_index, memdev->flags);
733 return true;
734 }
735
736 int nfit_get_smbios_id(u32 device_handle, u16 *flags)
737 {
738 struct acpi_nfit_memory_map *memdev;
739 struct acpi_nfit_desc *acpi_desc;
740 struct nfit_mem *nfit_mem;
741 u16 physical_id;
742
743 mutex_lock(&acpi_desc_lock);
744 list_for_each_entry(acpi_desc, &acpi_descs, list) {
745 mutex_lock(&acpi_desc->init_mutex);
746 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
747 memdev = __to_nfit_memdev(nfit_mem);
748 if (memdev->device_handle == device_handle) {
749 *flags = memdev->flags;
750 physical_id = memdev->physical_id;
751 mutex_unlock(&acpi_desc->init_mutex);
752 mutex_unlock(&acpi_desc_lock);
753 return physical_id;
754 }
755 }
756 mutex_unlock(&acpi_desc->init_mutex);
757 }
758 mutex_unlock(&acpi_desc_lock);
759
760 return -ENODEV;
761 }
762 EXPORT_SYMBOL_GPL(nfit_get_smbios_id);
763
764 /*
765 * An implementation may provide a truncated control region if no block windows
766 * are defined.
767 */
768 static size_t sizeof_dcr(struct acpi_nfit_control_region *dcr)
769 {
770 if (dcr->header.length < offsetof(struct acpi_nfit_control_region,
771 window_size))
772 return 0;
773 if (dcr->windows)
774 return sizeof(*dcr);
775 return offsetof(struct acpi_nfit_control_region, window_size);
776 }
777
778 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
779 struct nfit_table_prev *prev,
780 struct acpi_nfit_control_region *dcr)
781 {
782 struct device *dev = acpi_desc->dev;
783 struct nfit_dcr *nfit_dcr;
784
785 if (!sizeof_dcr(dcr))
786 return false;
787
788 list_for_each_entry(nfit_dcr, &prev->dcrs, list)
789 if (memcmp(nfit_dcr->dcr, dcr, sizeof_dcr(dcr)) == 0) {
790 list_move_tail(&nfit_dcr->list, &acpi_desc->dcrs);
791 return true;
792 }
793
794 nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr) + sizeof(*dcr),
795 GFP_KERNEL);
796 if (!nfit_dcr)
797 return false;
798 INIT_LIST_HEAD(&nfit_dcr->list);
799 memcpy(nfit_dcr->dcr, dcr, sizeof_dcr(dcr));
800 list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
801 dev_dbg(dev, "dcr index: %d windows: %d\n",
802 dcr->region_index, dcr->windows);
803 return true;
804 }
805
806 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
807 struct nfit_table_prev *prev,
808 struct acpi_nfit_data_region *bdw)
809 {
810 struct device *dev = acpi_desc->dev;
811 struct nfit_bdw *nfit_bdw;
812
813 if (bdw->header.length != sizeof(*bdw))
814 return false;
815 list_for_each_entry(nfit_bdw, &prev->bdws, list)
816 if (memcmp(nfit_bdw->bdw, bdw, sizeof(*bdw)) == 0) {
817 list_move_tail(&nfit_bdw->list, &acpi_desc->bdws);
818 return true;
819 }
820
821 nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw) + sizeof(*bdw),
822 GFP_KERNEL);
823 if (!nfit_bdw)
824 return false;
825 INIT_LIST_HEAD(&nfit_bdw->list);
826 memcpy(nfit_bdw->bdw, bdw, sizeof(*bdw));
827 list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
828 dev_dbg(dev, "bdw dcr: %d windows: %d\n",
829 bdw->region_index, bdw->windows);
830 return true;
831 }
832
833 static size_t sizeof_idt(struct acpi_nfit_interleave *idt)
834 {
835 if (idt->header.length < sizeof(*idt))
836 return 0;
837 return sizeof(*idt) + sizeof(u32) * (idt->line_count - 1);
838 }
839
840 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
841 struct nfit_table_prev *prev,
842 struct acpi_nfit_interleave *idt)
843 {
844 struct device *dev = acpi_desc->dev;
845 struct nfit_idt *nfit_idt;
846
847 if (!sizeof_idt(idt))
848 return false;
849
850 list_for_each_entry(nfit_idt, &prev->idts, list) {
851 if (sizeof_idt(nfit_idt->idt) != sizeof_idt(idt))
852 continue;
853
854 if (memcmp(nfit_idt->idt, idt, sizeof_idt(idt)) == 0) {
855 list_move_tail(&nfit_idt->list, &acpi_desc->idts);
856 return true;
857 }
858 }
859
860 nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt) + sizeof_idt(idt),
861 GFP_KERNEL);
862 if (!nfit_idt)
863 return false;
864 INIT_LIST_HEAD(&nfit_idt->list);
865 memcpy(nfit_idt->idt, idt, sizeof_idt(idt));
866 list_add_tail(&nfit_idt->list, &acpi_desc->idts);
867 dev_dbg(dev, "idt index: %d num_lines: %d\n",
868 idt->interleave_index, idt->line_count);
869 return true;
870 }
871
872 static size_t sizeof_flush(struct acpi_nfit_flush_address *flush)
873 {
874 if (flush->header.length < sizeof(*flush))
875 return 0;
876 return sizeof(*flush) + sizeof(u64) * (flush->hint_count - 1);
877 }
878
879 static bool add_flush(struct acpi_nfit_desc *acpi_desc,
880 struct nfit_table_prev *prev,
881 struct acpi_nfit_flush_address *flush)
882 {
883 struct device *dev = acpi_desc->dev;
884 struct nfit_flush *nfit_flush;
885
886 if (!sizeof_flush(flush))
887 return false;
888
889 list_for_each_entry(nfit_flush, &prev->flushes, list) {
890 if (sizeof_flush(nfit_flush->flush) != sizeof_flush(flush))
891 continue;
892
893 if (memcmp(nfit_flush->flush, flush,
894 sizeof_flush(flush)) == 0) {
895 list_move_tail(&nfit_flush->list, &acpi_desc->flushes);
896 return true;
897 }
898 }
899
900 nfit_flush = devm_kzalloc(dev, sizeof(*nfit_flush)
901 + sizeof_flush(flush), GFP_KERNEL);
902 if (!nfit_flush)
903 return false;
904 INIT_LIST_HEAD(&nfit_flush->list);
905 memcpy(nfit_flush->flush, flush, sizeof_flush(flush));
906 list_add_tail(&nfit_flush->list, &acpi_desc->flushes);
907 dev_dbg(dev, "nfit_flush handle: %d hint_count: %d\n",
908 flush->device_handle, flush->hint_count);
909 return true;
910 }
911
912 static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc,
913 struct acpi_nfit_capabilities *pcap)
914 {
915 struct device *dev = acpi_desc->dev;
916 u32 mask;
917
918 mask = (1 << (pcap->highest_capability + 1)) - 1;
919 acpi_desc->platform_cap = pcap->capabilities & mask;
920 dev_dbg(dev, "cap: %#x\n", acpi_desc->platform_cap);
921 return true;
922 }
923
924 static void *add_table(struct acpi_nfit_desc *acpi_desc,
925 struct nfit_table_prev *prev, void *table, const void *end)
926 {
927 struct device *dev = acpi_desc->dev;
928 struct acpi_nfit_header *hdr;
929 void *err = ERR_PTR(-ENOMEM);
930
931 if (table >= end)
932 return NULL;
933
934 hdr = table;
935 if (!hdr->length) {
936 dev_warn(dev, "found a zero length table '%d' parsing nfit\n",
937 hdr->type);
938 return NULL;
939 }
940
941 switch (hdr->type) {
942 case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
943 if (!add_spa(acpi_desc, prev, table))
944 return err;
945 break;
946 case ACPI_NFIT_TYPE_MEMORY_MAP:
947 if (!add_memdev(acpi_desc, prev, table))
948 return err;
949 break;
950 case ACPI_NFIT_TYPE_CONTROL_REGION:
951 if (!add_dcr(acpi_desc, prev, table))
952 return err;
953 break;
954 case ACPI_NFIT_TYPE_DATA_REGION:
955 if (!add_bdw(acpi_desc, prev, table))
956 return err;
957 break;
958 case ACPI_NFIT_TYPE_INTERLEAVE:
959 if (!add_idt(acpi_desc, prev, table))
960 return err;
961 break;
962 case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
963 if (!add_flush(acpi_desc, prev, table))
964 return err;
965 break;
966 case ACPI_NFIT_TYPE_SMBIOS:
967 dev_dbg(dev, "smbios\n");
968 break;
969 case ACPI_NFIT_TYPE_CAPABILITIES:
970 if (!add_platform_cap(acpi_desc, table))
971 return err;
972 break;
973 default:
974 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
975 break;
976 }
977
978 return table + hdr->length;
979 }
980
981 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
982 struct nfit_mem *nfit_mem)
983 {
984 u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
985 u16 dcr = nfit_mem->dcr->region_index;
986 struct nfit_spa *nfit_spa;
987
988 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
989 u16 range_index = nfit_spa->spa->range_index;
990 int type = nfit_spa_type(nfit_spa->spa);
991 struct nfit_memdev *nfit_memdev;
992
993 if (type != NFIT_SPA_BDW)
994 continue;
995
996 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
997 if (nfit_memdev->memdev->range_index != range_index)
998 continue;
999 if (nfit_memdev->memdev->device_handle != device_handle)
1000 continue;
1001 if (nfit_memdev->memdev->region_index != dcr)
1002 continue;
1003
1004 nfit_mem->spa_bdw = nfit_spa->spa;
1005 return;
1006 }
1007 }
1008
1009 dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
1010 nfit_mem->spa_dcr->range_index);
1011 nfit_mem->bdw = NULL;
1012 }
1013
1014 static void nfit_mem_init_bdw(struct acpi_nfit_desc *acpi_desc,
1015 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
1016 {
1017 u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
1018 struct nfit_memdev *nfit_memdev;
1019 struct nfit_bdw *nfit_bdw;
1020 struct nfit_idt *nfit_idt;
1021 u16 idt_idx, range_index;
1022
1023 list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
1024 if (nfit_bdw->bdw->region_index != dcr)
1025 continue;
1026 nfit_mem->bdw = nfit_bdw->bdw;
1027 break;
1028 }
1029
1030 if (!nfit_mem->bdw)
1031 return;
1032
1033 nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
1034
1035 if (!nfit_mem->spa_bdw)
1036 return;
1037
1038 range_index = nfit_mem->spa_bdw->range_index;
1039 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1040 if (nfit_memdev->memdev->range_index != range_index ||
1041 nfit_memdev->memdev->region_index != dcr)
1042 continue;
1043 nfit_mem->memdev_bdw = nfit_memdev->memdev;
1044 idt_idx = nfit_memdev->memdev->interleave_index;
1045 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1046 if (nfit_idt->idt->interleave_index != idt_idx)
1047 continue;
1048 nfit_mem->idt_bdw = nfit_idt->idt;
1049 break;
1050 }
1051 break;
1052 }
1053 }
1054
1055 static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc,
1056 struct acpi_nfit_system_address *spa)
1057 {
1058 struct nfit_mem *nfit_mem, *found;
1059 struct nfit_memdev *nfit_memdev;
1060 int type = spa ? nfit_spa_type(spa) : 0;
1061
1062 switch (type) {
1063 case NFIT_SPA_DCR:
1064 case NFIT_SPA_PM:
1065 break;
1066 default:
1067 if (spa)
1068 return 0;
1069 }
1070
1071 /*
1072 * This loop runs in two modes, when a dimm is mapped the loop
1073 * adds memdev associations to an existing dimm, or creates a
1074 * dimm. In the unmapped dimm case this loop sweeps for memdev
1075 * instances with an invalid / zero range_index and adds those
1076 * dimms without spa associations.
1077 */
1078 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1079 struct nfit_flush *nfit_flush;
1080 struct nfit_dcr *nfit_dcr;
1081 u32 device_handle;
1082 u16 dcr;
1083
1084 if (spa && nfit_memdev->memdev->range_index != spa->range_index)
1085 continue;
1086 if (!spa && nfit_memdev->memdev->range_index)
1087 continue;
1088 found = NULL;
1089 dcr = nfit_memdev->memdev->region_index;
1090 device_handle = nfit_memdev->memdev->device_handle;
1091 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1092 if (__to_nfit_memdev(nfit_mem)->device_handle
1093 == device_handle) {
1094 found = nfit_mem;
1095 break;
1096 }
1097
1098 if (found)
1099 nfit_mem = found;
1100 else {
1101 nfit_mem = devm_kzalloc(acpi_desc->dev,
1102 sizeof(*nfit_mem), GFP_KERNEL);
1103 if (!nfit_mem)
1104 return -ENOMEM;
1105 INIT_LIST_HEAD(&nfit_mem->list);
1106 nfit_mem->acpi_desc = acpi_desc;
1107 list_add(&nfit_mem->list, &acpi_desc->dimms);
1108 }
1109
1110 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1111 if (nfit_dcr->dcr->region_index != dcr)
1112 continue;
1113 /*
1114 * Record the control region for the dimm. For
1115 * the ACPI 6.1 case, where there are separate
1116 * control regions for the pmem vs blk
1117 * interfaces, be sure to record the extended
1118 * blk details.
1119 */
1120 if (!nfit_mem->dcr)
1121 nfit_mem->dcr = nfit_dcr->dcr;
1122 else if (nfit_mem->dcr->windows == 0
1123 && nfit_dcr->dcr->windows)
1124 nfit_mem->dcr = nfit_dcr->dcr;
1125 break;
1126 }
1127
1128 list_for_each_entry(nfit_flush, &acpi_desc->flushes, list) {
1129 struct acpi_nfit_flush_address *flush;
1130 u16 i;
1131
1132 if (nfit_flush->flush->device_handle != device_handle)
1133 continue;
1134 nfit_mem->nfit_flush = nfit_flush;
1135 flush = nfit_flush->flush;
1136 nfit_mem->flush_wpq = devm_kcalloc(acpi_desc->dev,
1137 flush->hint_count,
1138 sizeof(struct resource),
1139 GFP_KERNEL);
1140 if (!nfit_mem->flush_wpq)
1141 return -ENOMEM;
1142 for (i = 0; i < flush->hint_count; i++) {
1143 struct resource *res = &nfit_mem->flush_wpq[i];
1144
1145 res->start = flush->hint_address[i];
1146 res->end = res->start + 8 - 1;
1147 }
1148 break;
1149 }
1150
1151 if (dcr && !nfit_mem->dcr) {
1152 dev_err(acpi_desc->dev, "SPA %d missing DCR %d\n",
1153 spa->range_index, dcr);
1154 return -ENODEV;
1155 }
1156
1157 if (type == NFIT_SPA_DCR) {
1158 struct nfit_idt *nfit_idt;
1159 u16 idt_idx;
1160
1161 /* multiple dimms may share a SPA when interleaved */
1162 nfit_mem->spa_dcr = spa;
1163 nfit_mem->memdev_dcr = nfit_memdev->memdev;
1164 idt_idx = nfit_memdev->memdev->interleave_index;
1165 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
1166 if (nfit_idt->idt->interleave_index != idt_idx)
1167 continue;
1168 nfit_mem->idt_dcr = nfit_idt->idt;
1169 break;
1170 }
1171 nfit_mem_init_bdw(acpi_desc, nfit_mem, spa);
1172 } else if (type == NFIT_SPA_PM) {
1173 /*
1174 * A single dimm may belong to multiple SPA-PM
1175 * ranges, record at least one in addition to
1176 * any SPA-DCR range.
1177 */
1178 nfit_mem->memdev_pmem = nfit_memdev->memdev;
1179 } else
1180 nfit_mem->memdev_dcr = nfit_memdev->memdev;
1181 }
1182
1183 return 0;
1184 }
1185
1186 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
1187 {
1188 struct nfit_mem *a = container_of(_a, typeof(*a), list);
1189 struct nfit_mem *b = container_of(_b, typeof(*b), list);
1190 u32 handleA, handleB;
1191
1192 handleA = __to_nfit_memdev(a)->device_handle;
1193 handleB = __to_nfit_memdev(b)->device_handle;
1194 if (handleA < handleB)
1195 return -1;
1196 else if (handleA > handleB)
1197 return 1;
1198 return 0;
1199 }
1200
1201 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
1202 {
1203 struct nfit_spa *nfit_spa;
1204 int rc;
1205
1206
1207 /*
1208 * For each SPA-DCR or SPA-PMEM address range find its
1209 * corresponding MEMDEV(s). From each MEMDEV find the
1210 * corresponding DCR. Then, if we're operating on a SPA-DCR,
1211 * try to find a SPA-BDW and a corresponding BDW that references
1212 * the DCR. Throw it all into an nfit_mem object. Note, that
1213 * BDWs are optional.
1214 */
1215 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1216 rc = __nfit_mem_init(acpi_desc, nfit_spa->spa);
1217 if (rc)
1218 return rc;
1219 }
1220
1221 /*
1222 * If a DIMM has failed to be mapped into SPA there will be no
1223 * SPA entries above. Find and register all the unmapped DIMMs
1224 * for reporting and recovery purposes.
1225 */
1226 rc = __nfit_mem_init(acpi_desc, NULL);
1227 if (rc)
1228 return rc;
1229
1230 list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
1231
1232 return 0;
1233 }
1234
1235 static ssize_t bus_dsm_mask_show(struct device *dev,
1236 struct device_attribute *attr, char *buf)
1237 {
1238 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1239 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1240
1241 return sprintf(buf, "%#lx\n", nd_desc->bus_dsm_mask);
1242 }
1243 static struct device_attribute dev_attr_bus_dsm_mask =
1244 __ATTR(dsm_mask, 0444, bus_dsm_mask_show, NULL);
1245
1246 static ssize_t revision_show(struct device *dev,
1247 struct device_attribute *attr, char *buf)
1248 {
1249 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1250 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1251 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1252
1253 return sprintf(buf, "%d\n", acpi_desc->acpi_header.revision);
1254 }
1255 static DEVICE_ATTR_RO(revision);
1256
1257 static ssize_t hw_error_scrub_show(struct device *dev,
1258 struct device_attribute *attr, char *buf)
1259 {
1260 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1261 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1262 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1263
1264 return sprintf(buf, "%d\n", acpi_desc->scrub_mode);
1265 }
1266
1267 /*
1268 * The 'hw_error_scrub' attribute can have the following values written to it:
1269 * '0': Switch to the default mode where an exception will only insert
1270 * the address of the memory error into the poison and badblocks lists.
1271 * '1': Enable a full scrub to happen if an exception for a memory error is
1272 * received.
1273 */
1274 static ssize_t hw_error_scrub_store(struct device *dev,
1275 struct device_attribute *attr, const char *buf, size_t size)
1276 {
1277 struct nvdimm_bus_descriptor *nd_desc;
1278 ssize_t rc;
1279 long val;
1280
1281 rc = kstrtol(buf, 0, &val);
1282 if (rc)
1283 return rc;
1284
1285 nfit_device_lock(dev);
1286 nd_desc = dev_get_drvdata(dev);
1287 if (nd_desc) {
1288 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1289
1290 switch (val) {
1291 case HW_ERROR_SCRUB_ON:
1292 acpi_desc->scrub_mode = HW_ERROR_SCRUB_ON;
1293 break;
1294 case HW_ERROR_SCRUB_OFF:
1295 acpi_desc->scrub_mode = HW_ERROR_SCRUB_OFF;
1296 break;
1297 default:
1298 rc = -EINVAL;
1299 break;
1300 }
1301 }
1302 nfit_device_unlock(dev);
1303 if (rc)
1304 return rc;
1305 return size;
1306 }
1307 static DEVICE_ATTR_RW(hw_error_scrub);
1308
1309 /*
1310 * This shows the number of full Address Range Scrubs that have been
1311 * completed since driver load time. Userspace can wait on this using
1312 * select/poll etc. A '+' at the end indicates an ARS is in progress
1313 */
1314 static ssize_t scrub_show(struct device *dev,
1315 struct device_attribute *attr, char *buf)
1316 {
1317 struct nvdimm_bus_descriptor *nd_desc;
1318 struct acpi_nfit_desc *acpi_desc;
1319 ssize_t rc = -ENXIO;
1320 bool busy;
1321
1322 nfit_device_lock(dev);
1323 nd_desc = dev_get_drvdata(dev);
1324 if (!nd_desc) {
1325 nfit_device_unlock(dev);
1326 return rc;
1327 }
1328 acpi_desc = to_acpi_desc(nd_desc);
1329
1330 mutex_lock(&acpi_desc->init_mutex);
1331 busy = test_bit(ARS_BUSY, &acpi_desc->scrub_flags)
1332 && !test_bit(ARS_CANCEL, &acpi_desc->scrub_flags);
1333 rc = sprintf(buf, "%d%s", acpi_desc->scrub_count, busy ? "+\n" : "\n");
1334 /* Allow an admin to poll the busy state at a higher rate */
1335 if (busy && capable(CAP_SYS_RAWIO) && !test_and_set_bit(ARS_POLL,
1336 &acpi_desc->scrub_flags)) {
1337 acpi_desc->scrub_tmo = 1;
1338 mod_delayed_work(nfit_wq, &acpi_desc->dwork, HZ);
1339 }
1340
1341 mutex_unlock(&acpi_desc->init_mutex);
1342 nfit_device_unlock(dev);
1343 return rc;
1344 }
1345
1346 static ssize_t scrub_store(struct device *dev,
1347 struct device_attribute *attr, const char *buf, size_t size)
1348 {
1349 struct nvdimm_bus_descriptor *nd_desc;
1350 ssize_t rc;
1351 long val;
1352
1353 rc = kstrtol(buf, 0, &val);
1354 if (rc)
1355 return rc;
1356 if (val != 1)
1357 return -EINVAL;
1358
1359 nfit_device_lock(dev);
1360 nd_desc = dev_get_drvdata(dev);
1361 if (nd_desc) {
1362 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1363
1364 rc = acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
1365 }
1366 nfit_device_unlock(dev);
1367 if (rc)
1368 return rc;
1369 return size;
1370 }
1371 static DEVICE_ATTR_RW(scrub);
1372
1373 static bool ars_supported(struct nvdimm_bus *nvdimm_bus)
1374 {
1375 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1376 const unsigned long mask = 1 << ND_CMD_ARS_CAP | 1 << ND_CMD_ARS_START
1377 | 1 << ND_CMD_ARS_STATUS;
1378
1379 return (nd_desc->cmd_mask & mask) == mask;
1380 }
1381
1382 static umode_t nfit_visible(struct kobject *kobj, struct attribute *a, int n)
1383 {
1384 struct device *dev = container_of(kobj, struct device, kobj);
1385 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
1386
1387 if (a == &dev_attr_scrub.attr && !ars_supported(nvdimm_bus))
1388 return 0;
1389 return a->mode;
1390 }
1391
1392 static struct attribute *acpi_nfit_attributes[] = {
1393 &dev_attr_revision.attr,
1394 &dev_attr_scrub.attr,
1395 &dev_attr_hw_error_scrub.attr,
1396 &dev_attr_bus_dsm_mask.attr,
1397 NULL,
1398 };
1399
1400 static const struct attribute_group acpi_nfit_attribute_group = {
1401 .name = "nfit",
1402 .attrs = acpi_nfit_attributes,
1403 .is_visible = nfit_visible,
1404 };
1405
1406 static const struct attribute_group *acpi_nfit_attribute_groups[] = {
1407 &acpi_nfit_attribute_group,
1408 NULL,
1409 };
1410
1411 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
1412 {
1413 struct nvdimm *nvdimm = to_nvdimm(dev);
1414 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1415
1416 return __to_nfit_memdev(nfit_mem);
1417 }
1418
1419 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
1420 {
1421 struct nvdimm *nvdimm = to_nvdimm(dev);
1422 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1423
1424 return nfit_mem->dcr;
1425 }
1426
1427 static ssize_t handle_show(struct device *dev,
1428 struct device_attribute *attr, char *buf)
1429 {
1430 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1431
1432 return sprintf(buf, "%#x\n", memdev->device_handle);
1433 }
1434 static DEVICE_ATTR_RO(handle);
1435
1436 static ssize_t phys_id_show(struct device *dev,
1437 struct device_attribute *attr, char *buf)
1438 {
1439 struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
1440
1441 return sprintf(buf, "%#x\n", memdev->physical_id);
1442 }
1443 static DEVICE_ATTR_RO(phys_id);
1444
1445 static ssize_t vendor_show(struct device *dev,
1446 struct device_attribute *attr, char *buf)
1447 {
1448 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1449
1450 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->vendor_id));
1451 }
1452 static DEVICE_ATTR_RO(vendor);
1453
1454 static ssize_t rev_id_show(struct device *dev,
1455 struct device_attribute *attr, char *buf)
1456 {
1457 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1458
1459 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->revision_id));
1460 }
1461 static DEVICE_ATTR_RO(rev_id);
1462
1463 static ssize_t device_show(struct device *dev,
1464 struct device_attribute *attr, char *buf)
1465 {
1466 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1467
1468 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->device_id));
1469 }
1470 static DEVICE_ATTR_RO(device);
1471
1472 static ssize_t subsystem_vendor_show(struct device *dev,
1473 struct device_attribute *attr, char *buf)
1474 {
1475 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1476
1477 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_vendor_id));
1478 }
1479 static DEVICE_ATTR_RO(subsystem_vendor);
1480
1481 static ssize_t subsystem_rev_id_show(struct device *dev,
1482 struct device_attribute *attr, char *buf)
1483 {
1484 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1485
1486 return sprintf(buf, "0x%04x\n",
1487 be16_to_cpu(dcr->subsystem_revision_id));
1488 }
1489 static DEVICE_ATTR_RO(subsystem_rev_id);
1490
1491 static ssize_t subsystem_device_show(struct device *dev,
1492 struct device_attribute *attr, char *buf)
1493 {
1494 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1495
1496 return sprintf(buf, "0x%04x\n", be16_to_cpu(dcr->subsystem_device_id));
1497 }
1498 static DEVICE_ATTR_RO(subsystem_device);
1499
1500 static int num_nvdimm_formats(struct nvdimm *nvdimm)
1501 {
1502 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1503 int formats = 0;
1504
1505 if (nfit_mem->memdev_pmem)
1506 formats++;
1507 if (nfit_mem->memdev_bdw)
1508 formats++;
1509 return formats;
1510 }
1511
1512 static ssize_t format_show(struct device *dev,
1513 struct device_attribute *attr, char *buf)
1514 {
1515 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1516
1517 return sprintf(buf, "0x%04x\n", le16_to_cpu(dcr->code));
1518 }
1519 static DEVICE_ATTR_RO(format);
1520
1521 static ssize_t format1_show(struct device *dev,
1522 struct device_attribute *attr, char *buf)
1523 {
1524 u32 handle;
1525 ssize_t rc = -ENXIO;
1526 struct nfit_mem *nfit_mem;
1527 struct nfit_memdev *nfit_memdev;
1528 struct acpi_nfit_desc *acpi_desc;
1529 struct nvdimm *nvdimm = to_nvdimm(dev);
1530 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1531
1532 nfit_mem = nvdimm_provider_data(nvdimm);
1533 acpi_desc = nfit_mem->acpi_desc;
1534 handle = to_nfit_memdev(dev)->device_handle;
1535
1536 /* assumes DIMMs have at most 2 published interface codes */
1537 mutex_lock(&acpi_desc->init_mutex);
1538 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1539 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1540 struct nfit_dcr *nfit_dcr;
1541
1542 if (memdev->device_handle != handle)
1543 continue;
1544
1545 list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
1546 if (nfit_dcr->dcr->region_index != memdev->region_index)
1547 continue;
1548 if (nfit_dcr->dcr->code == dcr->code)
1549 continue;
1550 rc = sprintf(buf, "0x%04x\n",
1551 le16_to_cpu(nfit_dcr->dcr->code));
1552 break;
1553 }
1554 if (rc != ENXIO)
1555 break;
1556 }
1557 mutex_unlock(&acpi_desc->init_mutex);
1558 return rc;
1559 }
1560 static DEVICE_ATTR_RO(format1);
1561
1562 static ssize_t formats_show(struct device *dev,
1563 struct device_attribute *attr, char *buf)
1564 {
1565 struct nvdimm *nvdimm = to_nvdimm(dev);
1566
1567 return sprintf(buf, "%d\n", num_nvdimm_formats(nvdimm));
1568 }
1569 static DEVICE_ATTR_RO(formats);
1570
1571 static ssize_t serial_show(struct device *dev,
1572 struct device_attribute *attr, char *buf)
1573 {
1574 struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
1575
1576 return sprintf(buf, "0x%08x\n", be32_to_cpu(dcr->serial_number));
1577 }
1578 static DEVICE_ATTR_RO(serial);
1579
1580 static ssize_t family_show(struct device *dev,
1581 struct device_attribute *attr, char *buf)
1582 {
1583 struct nvdimm *nvdimm = to_nvdimm(dev);
1584 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1585
1586 if (nfit_mem->family < 0)
1587 return -ENXIO;
1588 return sprintf(buf, "%d\n", nfit_mem->family);
1589 }
1590 static DEVICE_ATTR_RO(family);
1591
1592 static ssize_t dsm_mask_show(struct device *dev,
1593 struct device_attribute *attr, char *buf)
1594 {
1595 struct nvdimm *nvdimm = to_nvdimm(dev);
1596 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1597
1598 if (nfit_mem->family < 0)
1599 return -ENXIO;
1600 return sprintf(buf, "%#lx\n", nfit_mem->dsm_mask);
1601 }
1602 static DEVICE_ATTR_RO(dsm_mask);
1603
1604 static ssize_t flags_show(struct device *dev,
1605 struct device_attribute *attr, char *buf)
1606 {
1607 struct nvdimm *nvdimm = to_nvdimm(dev);
1608 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1609 u16 flags = __to_nfit_memdev(nfit_mem)->flags;
1610
1611 if (test_bit(NFIT_MEM_DIRTY, &nfit_mem->flags))
1612 flags |= ACPI_NFIT_MEM_FLUSH_FAILED;
1613
1614 return sprintf(buf, "%s%s%s%s%s%s%s\n",
1615 flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
1616 flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
1617 flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
1618 flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
1619 flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "",
1620 flags & ACPI_NFIT_MEM_MAP_FAILED ? "map_fail " : "",
1621 flags & ACPI_NFIT_MEM_HEALTH_ENABLED ? "smart_notify " : "");
1622 }
1623 static DEVICE_ATTR_RO(flags);
1624
1625 static ssize_t id_show(struct device *dev,
1626 struct device_attribute *attr, char *buf)
1627 {
1628 struct nvdimm *nvdimm = to_nvdimm(dev);
1629 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1630
1631 return sprintf(buf, "%s\n", nfit_mem->id);
1632 }
1633 static DEVICE_ATTR_RO(id);
1634
1635 static ssize_t dirty_shutdown_show(struct device *dev,
1636 struct device_attribute *attr, char *buf)
1637 {
1638 struct nvdimm *nvdimm = to_nvdimm(dev);
1639 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1640
1641 return sprintf(buf, "%d\n", nfit_mem->dirty_shutdown);
1642 }
1643 static DEVICE_ATTR_RO(dirty_shutdown);
1644
1645 static struct attribute *acpi_nfit_dimm_attributes[] = {
1646 &dev_attr_handle.attr,
1647 &dev_attr_phys_id.attr,
1648 &dev_attr_vendor.attr,
1649 &dev_attr_device.attr,
1650 &dev_attr_rev_id.attr,
1651 &dev_attr_subsystem_vendor.attr,
1652 &dev_attr_subsystem_device.attr,
1653 &dev_attr_subsystem_rev_id.attr,
1654 &dev_attr_format.attr,
1655 &dev_attr_formats.attr,
1656 &dev_attr_format1.attr,
1657 &dev_attr_serial.attr,
1658 &dev_attr_flags.attr,
1659 &dev_attr_id.attr,
1660 &dev_attr_family.attr,
1661 &dev_attr_dsm_mask.attr,
1662 &dev_attr_dirty_shutdown.attr,
1663 NULL,
1664 };
1665
1666 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
1667 struct attribute *a, int n)
1668 {
1669 struct device *dev = container_of(kobj, struct device, kobj);
1670 struct nvdimm *nvdimm = to_nvdimm(dev);
1671 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
1672
1673 if (!to_nfit_dcr(dev)) {
1674 /* Without a dcr only the memdev attributes can be surfaced */
1675 if (a == &dev_attr_handle.attr || a == &dev_attr_phys_id.attr
1676 || a == &dev_attr_flags.attr
1677 || a == &dev_attr_family.attr
1678 || a == &dev_attr_dsm_mask.attr)
1679 return a->mode;
1680 return 0;
1681 }
1682
1683 if (a == &dev_attr_format1.attr && num_nvdimm_formats(nvdimm) <= 1)
1684 return 0;
1685
1686 if (!test_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags)
1687 && a == &dev_attr_dirty_shutdown.attr)
1688 return 0;
1689
1690 return a->mode;
1691 }
1692
1693 static const struct attribute_group acpi_nfit_dimm_attribute_group = {
1694 .name = "nfit",
1695 .attrs = acpi_nfit_dimm_attributes,
1696 .is_visible = acpi_nfit_dimm_attr_visible,
1697 };
1698
1699 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
1700 &acpi_nfit_dimm_attribute_group,
1701 NULL,
1702 };
1703
1704 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
1705 u32 device_handle)
1706 {
1707 struct nfit_mem *nfit_mem;
1708
1709 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
1710 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
1711 return nfit_mem->nvdimm;
1712
1713 return NULL;
1714 }
1715
1716 void __acpi_nvdimm_notify(struct device *dev, u32 event)
1717 {
1718 struct nfit_mem *nfit_mem;
1719 struct acpi_nfit_desc *acpi_desc;
1720
1721 dev_dbg(dev->parent, "%s: event: %d\n", dev_name(dev),
1722 event);
1723
1724 if (event != NFIT_NOTIFY_DIMM_HEALTH) {
1725 dev_dbg(dev->parent, "%s: unknown event: %d\n", dev_name(dev),
1726 event);
1727 return;
1728 }
1729
1730 acpi_desc = dev_get_drvdata(dev->parent);
1731 if (!acpi_desc)
1732 return;
1733
1734 /*
1735 * If we successfully retrieved acpi_desc, then we know nfit_mem data
1736 * is still valid.
1737 */
1738 nfit_mem = dev_get_drvdata(dev);
1739 if (nfit_mem && nfit_mem->flags_attr)
1740 sysfs_notify_dirent(nfit_mem->flags_attr);
1741 }
1742 EXPORT_SYMBOL_GPL(__acpi_nvdimm_notify);
1743
1744 static void acpi_nvdimm_notify(acpi_handle handle, u32 event, void *data)
1745 {
1746 struct acpi_device *adev = data;
1747 struct device *dev = &adev->dev;
1748
1749 nfit_device_lock(dev->parent);
1750 __acpi_nvdimm_notify(dev, event);
1751 nfit_device_unlock(dev->parent);
1752 }
1753
1754 static bool acpi_nvdimm_has_method(struct acpi_device *adev, char *method)
1755 {
1756 acpi_handle handle;
1757 acpi_status status;
1758
1759 status = acpi_get_handle(adev->handle, method, &handle);
1760
1761 if (ACPI_SUCCESS(status))
1762 return true;
1763 return false;
1764 }
1765
1766 __weak void nfit_intel_shutdown_status(struct nfit_mem *nfit_mem)
1767 {
1768 struct device *dev = &nfit_mem->adev->dev;
1769 struct nd_intel_smart smart = { 0 };
1770 union acpi_object in_buf = {
1771 .buffer.type = ACPI_TYPE_BUFFER,
1772 .buffer.length = 0,
1773 };
1774 union acpi_object in_obj = {
1775 .package.type = ACPI_TYPE_PACKAGE,
1776 .package.count = 1,
1777 .package.elements = &in_buf,
1778 };
1779 const u8 func = ND_INTEL_SMART;
1780 const guid_t *guid = to_nfit_uuid(nfit_mem->family);
1781 u8 revid = nfit_dsm_revid(nfit_mem->family, func);
1782 struct acpi_device *adev = nfit_mem->adev;
1783 acpi_handle handle = adev->handle;
1784 union acpi_object *out_obj;
1785
1786 if ((nfit_mem->dsm_mask & (1 << func)) == 0)
1787 return;
1788
1789 out_obj = acpi_evaluate_dsm(handle, guid, revid, func, &in_obj);
1790 if (!out_obj || out_obj->type != ACPI_TYPE_BUFFER
1791 || out_obj->buffer.length < sizeof(smart)) {
1792 dev_dbg(dev->parent, "%s: failed to retrieve initial health\n",
1793 dev_name(dev));
1794 ACPI_FREE(out_obj);
1795 return;
1796 }
1797 memcpy(&smart, out_obj->buffer.pointer, sizeof(smart));
1798 ACPI_FREE(out_obj);
1799
1800 if (smart.flags & ND_INTEL_SMART_SHUTDOWN_VALID) {
1801 if (smart.shutdown_state)
1802 set_bit(NFIT_MEM_DIRTY, &nfit_mem->flags);
1803 }
1804
1805 if (smart.flags & ND_INTEL_SMART_SHUTDOWN_COUNT_VALID) {
1806 set_bit(NFIT_MEM_DIRTY_COUNT, &nfit_mem->flags);
1807 nfit_mem->dirty_shutdown = smart.shutdown_count;
1808 }
1809 }
1810
1811 static void populate_shutdown_status(struct nfit_mem *nfit_mem)
1812 {
1813 /*
1814 * For DIMMs that provide a dynamic facility to retrieve a
1815 * dirty-shutdown status and/or a dirty-shutdown count, cache
1816 * these values in nfit_mem.
1817 */
1818 if (nfit_mem->family == NVDIMM_FAMILY_INTEL)
1819 nfit_intel_shutdown_status(nfit_mem);
1820 }
1821
1822 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
1823 struct nfit_mem *nfit_mem, u32 device_handle)
1824 {
1825 struct acpi_device *adev, *adev_dimm;
1826 struct device *dev = acpi_desc->dev;
1827 unsigned long dsm_mask, label_mask;
1828 const guid_t *guid;
1829 int i;
1830 int family = -1;
1831 struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
1832
1833 /* nfit test assumes 1:1 relationship between commands and dsms */
1834 nfit_mem->dsm_mask = acpi_desc->dimm_cmd_force_en;
1835 nfit_mem->family = NVDIMM_FAMILY_INTEL;
1836
1837 if (dcr->valid_fields & ACPI_NFIT_CONTROL_MFG_INFO_VALID)
1838 sprintf(nfit_mem->id, "%04x-%02x-%04x-%08x",
1839 be16_to_cpu(dcr->vendor_id),
1840 dcr->manufacturing_location,
1841 be16_to_cpu(dcr->manufacturing_date),
1842 be32_to_cpu(dcr->serial_number));
1843 else
1844 sprintf(nfit_mem->id, "%04x-%08x",
1845 be16_to_cpu(dcr->vendor_id),
1846 be32_to_cpu(dcr->serial_number));
1847
1848 adev = to_acpi_dev(acpi_desc);
1849 if (!adev) {
1850 /* unit test case */
1851 populate_shutdown_status(nfit_mem);
1852 return 0;
1853 }
1854
1855 adev_dimm = acpi_find_child_device(adev, device_handle, false);
1856 nfit_mem->adev = adev_dimm;
1857 if (!adev_dimm) {
1858 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
1859 device_handle);
1860 return force_enable_dimms ? 0 : -ENODEV;
1861 }
1862
1863 if (ACPI_FAILURE(acpi_install_notify_handler(adev_dimm->handle,
1864 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify, adev_dimm))) {
1865 dev_err(dev, "%s: notification registration failed\n",
1866 dev_name(&adev_dimm->dev));
1867 return -ENXIO;
1868 }
1869 /*
1870 * Record nfit_mem for the notification path to track back to
1871 * the nfit sysfs attributes for this dimm device object.
1872 */
1873 dev_set_drvdata(&adev_dimm->dev, nfit_mem);
1874
1875 /*
1876 * There are 4 "legacy" NVDIMM command sets
1877 * (NVDIMM_FAMILY_{INTEL,MSFT,HPE1,HPE2}) that were created before
1878 * an EFI working group was established to constrain this
1879 * proliferation. The nfit driver probes for the supported command
1880 * set by GUID. Note, if you're a platform developer looking to add
1881 * a new command set to this probe, consider using an existing set,
1882 * or otherwise seek approval to publish the command set at
1883 * http://www.uefi.org/RFIC_LIST.
1884 *
1885 * Note, that checking for function0 (bit0) tells us if any commands
1886 * are reachable through this GUID.
1887 */
1888 for (i = 0; i <= NVDIMM_FAMILY_MAX; i++)
1889 if (acpi_check_dsm(adev_dimm->handle, to_nfit_uuid(i), 1, 1))
1890 if (family < 0 || i == default_dsm_family)
1891 family = i;
1892
1893 /* limit the supported commands to those that are publicly documented */
1894 nfit_mem->family = family;
1895 if (override_dsm_mask && !disable_vendor_specific)
1896 dsm_mask = override_dsm_mask;
1897 else if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
1898 dsm_mask = NVDIMM_INTEL_CMDMASK;
1899 if (disable_vendor_specific)
1900 dsm_mask &= ~(1 << ND_CMD_VENDOR);
1901 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE1) {
1902 dsm_mask = 0x1c3c76;
1903 } else if (nfit_mem->family == NVDIMM_FAMILY_HPE2) {
1904 dsm_mask = 0x1fe;
1905 if (disable_vendor_specific)
1906 dsm_mask &= ~(1 << 8);
1907 } else if (nfit_mem->family == NVDIMM_FAMILY_MSFT) {
1908 dsm_mask = 0xffffffff;
1909 } else if (nfit_mem->family == NVDIMM_FAMILY_HYPERV) {
1910 dsm_mask = 0x1f;
1911 } else {
1912 dev_dbg(dev, "unknown dimm command family\n");
1913 nfit_mem->family = -1;
1914 /* DSMs are optional, continue loading the driver... */
1915 return 0;
1916 }
1917
1918 /*
1919 * Function 0 is the command interrogation function, don't
1920 * export it to potential userspace use, and enable it to be
1921 * used as an error value in acpi_nfit_ctl().
1922 */
1923 dsm_mask &= ~1UL;
1924
1925 guid = to_nfit_uuid(nfit_mem->family);
1926 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
1927 if (acpi_check_dsm(adev_dimm->handle, guid,
1928 nfit_dsm_revid(nfit_mem->family, i),
1929 1ULL << i))
1930 set_bit(i, &nfit_mem->dsm_mask);
1931
1932 /*
1933 * Prefer the NVDIMM_FAMILY_INTEL label read commands if present
1934 * due to their better semantics handling locked capacity.
1935 */
1936 label_mask = 1 << ND_CMD_GET_CONFIG_SIZE | 1 << ND_CMD_GET_CONFIG_DATA
1937 | 1 << ND_CMD_SET_CONFIG_DATA;
1938 if (family == NVDIMM_FAMILY_INTEL
1939 && (dsm_mask & label_mask) == label_mask)
1940 /* skip _LS{I,R,W} enabling */;
1941 else {
1942 if (acpi_nvdimm_has_method(adev_dimm, "_LSI")
1943 && acpi_nvdimm_has_method(adev_dimm, "_LSR")) {
1944 dev_dbg(dev, "%s: has _LSR\n", dev_name(&adev_dimm->dev));
1945 set_bit(NFIT_MEM_LSR, &nfit_mem->flags);
1946 }
1947
1948 if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)
1949 && acpi_nvdimm_has_method(adev_dimm, "_LSW")) {
1950 dev_dbg(dev, "%s: has _LSW\n", dev_name(&adev_dimm->dev));
1951 set_bit(NFIT_MEM_LSW, &nfit_mem->flags);
1952 }
1953
1954 /*
1955 * Quirk read-only label configurations to preserve
1956 * access to label-less namespaces by default.
1957 */
1958 if (!test_bit(NFIT_MEM_LSW, &nfit_mem->flags)
1959 && !force_labels) {
1960 dev_dbg(dev, "%s: No _LSW, disable labels\n",
1961 dev_name(&adev_dimm->dev));
1962 clear_bit(NFIT_MEM_LSR, &nfit_mem->flags);
1963 } else
1964 dev_dbg(dev, "%s: Force enable labels\n",
1965 dev_name(&adev_dimm->dev));
1966 }
1967
1968 populate_shutdown_status(nfit_mem);
1969
1970 return 0;
1971 }
1972
1973 static void shutdown_dimm_notify(void *data)
1974 {
1975 struct acpi_nfit_desc *acpi_desc = data;
1976 struct nfit_mem *nfit_mem;
1977
1978 mutex_lock(&acpi_desc->init_mutex);
1979 /*
1980 * Clear out the nfit_mem->flags_attr and shut down dimm event
1981 * notifications.
1982 */
1983 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
1984 struct acpi_device *adev_dimm = nfit_mem->adev;
1985
1986 if (nfit_mem->flags_attr) {
1987 sysfs_put(nfit_mem->flags_attr);
1988 nfit_mem->flags_attr = NULL;
1989 }
1990 if (adev_dimm) {
1991 acpi_remove_notify_handler(adev_dimm->handle,
1992 ACPI_DEVICE_NOTIFY, acpi_nvdimm_notify);
1993 dev_set_drvdata(&adev_dimm->dev, NULL);
1994 }
1995 }
1996 mutex_unlock(&acpi_desc->init_mutex);
1997 }
1998
1999 static const struct nvdimm_security_ops *acpi_nfit_get_security_ops(int family)
2000 {
2001 switch (family) {
2002 case NVDIMM_FAMILY_INTEL:
2003 return intel_security_ops;
2004 default:
2005 return NULL;
2006 }
2007 }
2008
2009 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
2010 {
2011 struct nfit_mem *nfit_mem;
2012 int dimm_count = 0, rc;
2013 struct nvdimm *nvdimm;
2014
2015 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2016 struct acpi_nfit_flush_address *flush;
2017 unsigned long flags = 0, cmd_mask;
2018 struct nfit_memdev *nfit_memdev;
2019 u32 device_handle;
2020 u16 mem_flags;
2021
2022 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
2023 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
2024 if (nvdimm) {
2025 dimm_count++;
2026 continue;
2027 }
2028
2029 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
2030 set_bit(NDD_ALIASING, &flags);
2031
2032 /* collate flags across all memdevs for this dimm */
2033 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2034 struct acpi_nfit_memory_map *dimm_memdev;
2035
2036 dimm_memdev = __to_nfit_memdev(nfit_mem);
2037 if (dimm_memdev->device_handle
2038 != nfit_memdev->memdev->device_handle)
2039 continue;
2040 dimm_memdev->flags |= nfit_memdev->memdev->flags;
2041 }
2042
2043 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
2044 if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
2045 set_bit(NDD_UNARMED, &flags);
2046
2047 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
2048 if (rc)
2049 continue;
2050
2051 /*
2052 * TODO: provide translation for non-NVDIMM_FAMILY_INTEL
2053 * devices (i.e. from nd_cmd to acpi_dsm) to standardize the
2054 * userspace interface.
2055 */
2056 cmd_mask = 1UL << ND_CMD_CALL;
2057 if (nfit_mem->family == NVDIMM_FAMILY_INTEL) {
2058 /*
2059 * These commands have a 1:1 correspondence
2060 * between DSM payload and libnvdimm ioctl
2061 * payload format.
2062 */
2063 cmd_mask |= nfit_mem->dsm_mask & NVDIMM_STANDARD_CMDMASK;
2064 }
2065
2066 /* Quirk to ignore LOCAL for labels on HYPERV DIMMs */
2067 if (nfit_mem->family == NVDIMM_FAMILY_HYPERV)
2068 set_bit(NDD_NOBLK, &flags);
2069
2070 if (test_bit(NFIT_MEM_LSR, &nfit_mem->flags)) {
2071 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
2072 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
2073 }
2074 if (test_bit(NFIT_MEM_LSW, &nfit_mem->flags))
2075 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
2076
2077 flush = nfit_mem->nfit_flush ? nfit_mem->nfit_flush->flush
2078 : NULL;
2079 nvdimm = __nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
2080 acpi_nfit_dimm_attribute_groups,
2081 flags, cmd_mask, flush ? flush->hint_count : 0,
2082 nfit_mem->flush_wpq, &nfit_mem->id[0],
2083 acpi_nfit_get_security_ops(nfit_mem->family));
2084 if (!nvdimm)
2085 return -ENOMEM;
2086
2087 nfit_mem->nvdimm = nvdimm;
2088 dimm_count++;
2089
2090 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
2091 continue;
2092
2093 dev_err(acpi_desc->dev, "Error found in NVDIMM %s flags:%s%s%s%s%s\n",
2094 nvdimm_name(nvdimm),
2095 mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
2096 mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
2097 mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
2098 mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "",
2099 mem_flags & ACPI_NFIT_MEM_MAP_FAILED ? " map_fail" : "");
2100
2101 }
2102
2103 rc = nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
2104 if (rc)
2105 return rc;
2106
2107 /*
2108 * Now that dimms are successfully registered, and async registration
2109 * is flushed, attempt to enable event notification.
2110 */
2111 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
2112 struct kernfs_node *nfit_kernfs;
2113
2114 nvdimm = nfit_mem->nvdimm;
2115 if (!nvdimm)
2116 continue;
2117
2118 nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
2119 if (nfit_kernfs)
2120 nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
2121 "flags");
2122 sysfs_put(nfit_kernfs);
2123 if (!nfit_mem->flags_attr)
2124 dev_warn(acpi_desc->dev, "%s: notifications disabled\n",
2125 nvdimm_name(nvdimm));
2126 }
2127
2128 return devm_add_action_or_reset(acpi_desc->dev, shutdown_dimm_notify,
2129 acpi_desc);
2130 }
2131
2132 /*
2133 * These constants are private because there are no kernel consumers of
2134 * these commands.
2135 */
2136 enum nfit_aux_cmds {
2137 NFIT_CMD_TRANSLATE_SPA = 5,
2138 NFIT_CMD_ARS_INJECT_SET = 7,
2139 NFIT_CMD_ARS_INJECT_CLEAR = 8,
2140 NFIT_CMD_ARS_INJECT_GET = 9,
2141 };
2142
2143 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
2144 {
2145 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2146 const guid_t *guid = to_nfit_uuid(NFIT_DEV_BUS);
2147 struct acpi_device *adev;
2148 unsigned long dsm_mask;
2149 int i;
2150
2151 nd_desc->cmd_mask = acpi_desc->bus_cmd_force_en;
2152 nd_desc->bus_dsm_mask = acpi_desc->bus_nfit_cmd_force_en;
2153 adev = to_acpi_dev(acpi_desc);
2154 if (!adev)
2155 return;
2156
2157 for (i = ND_CMD_ARS_CAP; i <= ND_CMD_CLEAR_ERROR; i++)
2158 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2159 set_bit(i, &nd_desc->cmd_mask);
2160 set_bit(ND_CMD_CALL, &nd_desc->cmd_mask);
2161
2162 dsm_mask =
2163 (1 << ND_CMD_ARS_CAP) |
2164 (1 << ND_CMD_ARS_START) |
2165 (1 << ND_CMD_ARS_STATUS) |
2166 (1 << ND_CMD_CLEAR_ERROR) |
2167 (1 << NFIT_CMD_TRANSLATE_SPA) |
2168 (1 << NFIT_CMD_ARS_INJECT_SET) |
2169 (1 << NFIT_CMD_ARS_INJECT_CLEAR) |
2170 (1 << NFIT_CMD_ARS_INJECT_GET);
2171 for_each_set_bit(i, &dsm_mask, BITS_PER_LONG)
2172 if (acpi_check_dsm(adev->handle, guid, 1, 1ULL << i))
2173 set_bit(i, &nd_desc->bus_dsm_mask);
2174 }
2175
2176 static ssize_t range_index_show(struct device *dev,
2177 struct device_attribute *attr, char *buf)
2178 {
2179 struct nd_region *nd_region = to_nd_region(dev);
2180 struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
2181
2182 return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
2183 }
2184 static DEVICE_ATTR_RO(range_index);
2185
2186 static struct attribute *acpi_nfit_region_attributes[] = {
2187 &dev_attr_range_index.attr,
2188 NULL,
2189 };
2190
2191 static const struct attribute_group acpi_nfit_region_attribute_group = {
2192 .name = "nfit",
2193 .attrs = acpi_nfit_region_attributes,
2194 };
2195
2196 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
2197 &acpi_nfit_region_attribute_group,
2198 NULL,
2199 };
2200
2201 /* enough info to uniquely specify an interleave set */
2202 struct nfit_set_info {
2203 struct nfit_set_info_map {
2204 u64 region_offset;
2205 u32 serial_number;
2206 u32 pad;
2207 } mapping[0];
2208 };
2209
2210 struct nfit_set_info2 {
2211 struct nfit_set_info_map2 {
2212 u64 region_offset;
2213 u32 serial_number;
2214 u16 vendor_id;
2215 u16 manufacturing_date;
2216 u8 manufacturing_location;
2217 u8 reserved[31];
2218 } mapping[0];
2219 };
2220
2221 static size_t sizeof_nfit_set_info(int num_mappings)
2222 {
2223 return sizeof(struct nfit_set_info)
2224 + num_mappings * sizeof(struct nfit_set_info_map);
2225 }
2226
2227 static size_t sizeof_nfit_set_info2(int num_mappings)
2228 {
2229 return sizeof(struct nfit_set_info2)
2230 + num_mappings * sizeof(struct nfit_set_info_map2);
2231 }
2232
2233 static int cmp_map_compat(const void *m0, const void *m1)
2234 {
2235 const struct nfit_set_info_map *map0 = m0;
2236 const struct nfit_set_info_map *map1 = m1;
2237
2238 return memcmp(&map0->region_offset, &map1->region_offset,
2239 sizeof(u64));
2240 }
2241
2242 static int cmp_map(const void *m0, const void *m1)
2243 {
2244 const struct nfit_set_info_map *map0 = m0;
2245 const struct nfit_set_info_map *map1 = m1;
2246
2247 if (map0->region_offset < map1->region_offset)
2248 return -1;
2249 else if (map0->region_offset > map1->region_offset)
2250 return 1;
2251 return 0;
2252 }
2253
2254 static int cmp_map2(const void *m0, const void *m1)
2255 {
2256 const struct nfit_set_info_map2 *map0 = m0;
2257 const struct nfit_set_info_map2 *map1 = m1;
2258
2259 if (map0->region_offset < map1->region_offset)
2260 return -1;
2261 else if (map0->region_offset > map1->region_offset)
2262 return 1;
2263 return 0;
2264 }
2265
2266 /* Retrieve the nth entry referencing this spa */
2267 static struct acpi_nfit_memory_map *memdev_from_spa(
2268 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
2269 {
2270 struct nfit_memdev *nfit_memdev;
2271
2272 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
2273 if (nfit_memdev->memdev->range_index == range_index)
2274 if (n-- == 0)
2275 return nfit_memdev->memdev;
2276 return NULL;
2277 }
2278
2279 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
2280 struct nd_region_desc *ndr_desc,
2281 struct acpi_nfit_system_address *spa)
2282 {
2283 struct device *dev = acpi_desc->dev;
2284 struct nd_interleave_set *nd_set;
2285 u16 nr = ndr_desc->num_mappings;
2286 struct nfit_set_info2 *info2;
2287 struct nfit_set_info *info;
2288 int i;
2289
2290 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
2291 if (!nd_set)
2292 return -ENOMEM;
2293 guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid);
2294
2295 info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
2296 if (!info)
2297 return -ENOMEM;
2298
2299 info2 = devm_kzalloc(dev, sizeof_nfit_set_info2(nr), GFP_KERNEL);
2300 if (!info2)
2301 return -ENOMEM;
2302
2303 for (i = 0; i < nr; i++) {
2304 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
2305 struct nfit_set_info_map *map = &info->mapping[i];
2306 struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2307 struct nvdimm *nvdimm = mapping->nvdimm;
2308 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2309 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
2310 spa->range_index, i);
2311 struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2312
2313 if (!memdev || !nfit_mem->dcr) {
2314 dev_err(dev, "%s: failed to find DCR\n", __func__);
2315 return -ENODEV;
2316 }
2317
2318 map->region_offset = memdev->region_offset;
2319 map->serial_number = dcr->serial_number;
2320
2321 map2->region_offset = memdev->region_offset;
2322 map2->serial_number = dcr->serial_number;
2323 map2->vendor_id = dcr->vendor_id;
2324 map2->manufacturing_date = dcr->manufacturing_date;
2325 map2->manufacturing_location = dcr->manufacturing_location;
2326 }
2327
2328 /* v1.1 namespaces */
2329 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2330 cmp_map, NULL);
2331 nd_set->cookie1 = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2332
2333 /* v1.2 namespaces */
2334 sort(&info2->mapping[0], nr, sizeof(struct nfit_set_info_map2),
2335 cmp_map2, NULL);
2336 nd_set->cookie2 = nd_fletcher64(info2, sizeof_nfit_set_info2(nr), 0);
2337
2338 /* support v1.1 namespaces created with the wrong sort order */
2339 sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
2340 cmp_map_compat, NULL);
2341 nd_set->altcookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
2342
2343 /* record the result of the sort for the mapping position */
2344 for (i = 0; i < nr; i++) {
2345 struct nfit_set_info_map2 *map2 = &info2->mapping[i];
2346 int j;
2347
2348 for (j = 0; j < nr; j++) {
2349 struct nd_mapping_desc *mapping = &ndr_desc->mapping[j];
2350 struct nvdimm *nvdimm = mapping->nvdimm;
2351 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
2352 struct acpi_nfit_control_region *dcr = nfit_mem->dcr;
2353
2354 if (map2->serial_number == dcr->serial_number &&
2355 map2->vendor_id == dcr->vendor_id &&
2356 map2->manufacturing_date == dcr->manufacturing_date &&
2357 map2->manufacturing_location
2358 == dcr->manufacturing_location) {
2359 mapping->position = i;
2360 break;
2361 }
2362 }
2363 }
2364
2365 ndr_desc->nd_set = nd_set;
2366 devm_kfree(dev, info);
2367 devm_kfree(dev, info2);
2368
2369 return 0;
2370 }
2371
2372 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
2373 {
2374 struct acpi_nfit_interleave *idt = mmio->idt;
2375 u32 sub_line_offset, line_index, line_offset;
2376 u64 line_no, table_skip_count, table_offset;
2377
2378 line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
2379 table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
2380 line_offset = idt->line_offset[line_index]
2381 * mmio->line_size;
2382 table_offset = table_skip_count * mmio->table_size;
2383
2384 return mmio->base_offset + line_offset + table_offset + sub_line_offset;
2385 }
2386
2387 static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
2388 {
2389 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2390 u64 offset = nfit_blk->stat_offset + mmio->size * bw;
2391 const u32 STATUS_MASK = 0x80000037;
2392
2393 if (mmio->num_lines)
2394 offset = to_interleave_offset(offset, mmio);
2395
2396 return readl(mmio->addr.base + offset) & STATUS_MASK;
2397 }
2398
2399 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
2400 resource_size_t dpa, unsigned int len, unsigned int write)
2401 {
2402 u64 cmd, offset;
2403 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
2404
2405 enum {
2406 BCW_OFFSET_MASK = (1ULL << 48)-1,
2407 BCW_LEN_SHIFT = 48,
2408 BCW_LEN_MASK = (1ULL << 8) - 1,
2409 BCW_CMD_SHIFT = 56,
2410 };
2411
2412 cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
2413 len = len >> L1_CACHE_SHIFT;
2414 cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
2415 cmd |= ((u64) write) << BCW_CMD_SHIFT;
2416
2417 offset = nfit_blk->cmd_offset + mmio->size * bw;
2418 if (mmio->num_lines)
2419 offset = to_interleave_offset(offset, mmio);
2420
2421 writeq(cmd, mmio->addr.base + offset);
2422 nvdimm_flush(nfit_blk->nd_region, NULL);
2423
2424 if (nfit_blk->dimm_flags & NFIT_BLK_DCR_LATCH)
2425 readq(mmio->addr.base + offset);
2426 }
2427
2428 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
2429 resource_size_t dpa, void *iobuf, size_t len, int rw,
2430 unsigned int lane)
2431 {
2432 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2433 unsigned int copied = 0;
2434 u64 base_offset;
2435 int rc;
2436
2437 base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
2438 + lane * mmio->size;
2439 write_blk_ctl(nfit_blk, lane, dpa, len, rw);
2440 while (len) {
2441 unsigned int c;
2442 u64 offset;
2443
2444 if (mmio->num_lines) {
2445 u32 line_offset;
2446
2447 offset = to_interleave_offset(base_offset + copied,
2448 mmio);
2449 div_u64_rem(offset, mmio->line_size, &line_offset);
2450 c = min_t(size_t, len, mmio->line_size - line_offset);
2451 } else {
2452 offset = base_offset + nfit_blk->bdw_offset;
2453 c = len;
2454 }
2455
2456 if (rw)
2457 memcpy_flushcache(mmio->addr.aperture + offset, iobuf + copied, c);
2458 else {
2459 if (nfit_blk->dimm_flags & NFIT_BLK_READ_FLUSH)
2460 arch_invalidate_pmem((void __force *)
2461 mmio->addr.aperture + offset, c);
2462
2463 memcpy(iobuf + copied, mmio->addr.aperture + offset, c);
2464 }
2465
2466 copied += c;
2467 len -= c;
2468 }
2469
2470 if (rw)
2471 nvdimm_flush(nfit_blk->nd_region, NULL);
2472
2473 rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
2474 return rc;
2475 }
2476
2477 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
2478 resource_size_t dpa, void *iobuf, u64 len, int rw)
2479 {
2480 struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
2481 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
2482 struct nd_region *nd_region = nfit_blk->nd_region;
2483 unsigned int lane, copied = 0;
2484 int rc = 0;
2485
2486 lane = nd_region_acquire_lane(nd_region);
2487 while (len) {
2488 u64 c = min(len, mmio->size);
2489
2490 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
2491 iobuf + copied, c, rw, lane);
2492 if (rc)
2493 break;
2494
2495 copied += c;
2496 len -= c;
2497 }
2498 nd_region_release_lane(nd_region, lane);
2499
2500 return rc;
2501 }
2502
2503 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
2504 struct acpi_nfit_interleave *idt, u16 interleave_ways)
2505 {
2506 if (idt) {
2507 mmio->num_lines = idt->line_count;
2508 mmio->line_size = idt->line_size;
2509 if (interleave_ways == 0)
2510 return -ENXIO;
2511 mmio->table_size = mmio->num_lines * interleave_ways
2512 * mmio->line_size;
2513 }
2514
2515 return 0;
2516 }
2517
2518 static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc,
2519 struct nvdimm *nvdimm, struct nfit_blk *nfit_blk)
2520 {
2521 struct nd_cmd_dimm_flags flags;
2522 int rc;
2523
2524 memset(&flags, 0, sizeof(flags));
2525 rc = nd_desc->ndctl(nd_desc, nvdimm, ND_CMD_DIMM_FLAGS, &flags,
2526 sizeof(flags), NULL);
2527
2528 if (rc >= 0 && flags.status == 0)
2529 nfit_blk->dimm_flags = flags.flags;
2530 else if (rc == -ENOTTY) {
2531 /* fall back to a conservative default */
2532 nfit_blk->dimm_flags = NFIT_BLK_DCR_LATCH | NFIT_BLK_READ_FLUSH;
2533 rc = 0;
2534 } else
2535 rc = -ENXIO;
2536
2537 return rc;
2538 }
2539
2540 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
2541 struct device *dev)
2542 {
2543 struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
2544 struct nd_blk_region *ndbr = to_nd_blk_region(dev);
2545 struct nfit_blk_mmio *mmio;
2546 struct nfit_blk *nfit_blk;
2547 struct nfit_mem *nfit_mem;
2548 struct nvdimm *nvdimm;
2549 int rc;
2550
2551 nvdimm = nd_blk_region_to_dimm(ndbr);
2552 nfit_mem = nvdimm_provider_data(nvdimm);
2553 if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
2554 dev_dbg(dev, "missing%s%s%s\n",
2555 nfit_mem ? "" : " nfit_mem",
2556 (nfit_mem && nfit_mem->dcr) ? "" : " dcr",
2557 (nfit_mem && nfit_mem->bdw) ? "" : " bdw");
2558 return -ENXIO;
2559 }
2560
2561 nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
2562 if (!nfit_blk)
2563 return -ENOMEM;
2564 nd_blk_region_set_provider_data(ndbr, nfit_blk);
2565 nfit_blk->nd_region = to_nd_region(dev);
2566
2567 /* map block aperture memory */
2568 nfit_blk->bdw_offset = nfit_mem->bdw->offset;
2569 mmio = &nfit_blk->mmio[BDW];
2570 mmio->addr.base = devm_nvdimm_memremap(dev, nfit_mem->spa_bdw->address,
2571 nfit_mem->spa_bdw->length, nd_blk_memremap_flags(ndbr));
2572 if (!mmio->addr.base) {
2573 dev_dbg(dev, "%s failed to map bdw\n",
2574 nvdimm_name(nvdimm));
2575 return -ENOMEM;
2576 }
2577 mmio->size = nfit_mem->bdw->size;
2578 mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
2579 mmio->idt = nfit_mem->idt_bdw;
2580 mmio->spa = nfit_mem->spa_bdw;
2581 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
2582 nfit_mem->memdev_bdw->interleave_ways);
2583 if (rc) {
2584 dev_dbg(dev, "%s failed to init bdw interleave\n",
2585 nvdimm_name(nvdimm));
2586 return rc;
2587 }
2588
2589 /* map block control memory */
2590 nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
2591 nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
2592 mmio = &nfit_blk->mmio[DCR];
2593 mmio->addr.base = devm_nvdimm_ioremap(dev, nfit_mem->spa_dcr->address,
2594 nfit_mem->spa_dcr->length);
2595 if (!mmio->addr.base) {
2596 dev_dbg(dev, "%s failed to map dcr\n",
2597 nvdimm_name(nvdimm));
2598 return -ENOMEM;
2599 }
2600 mmio->size = nfit_mem->dcr->window_size;
2601 mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
2602 mmio->idt = nfit_mem->idt_dcr;
2603 mmio->spa = nfit_mem->spa_dcr;
2604 rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
2605 nfit_mem->memdev_dcr->interleave_ways);
2606 if (rc) {
2607 dev_dbg(dev, "%s failed to init dcr interleave\n",
2608 nvdimm_name(nvdimm));
2609 return rc;
2610 }
2611
2612 rc = acpi_nfit_blk_get_flags(nd_desc, nvdimm, nfit_blk);
2613 if (rc < 0) {
2614 dev_dbg(dev, "%s failed get DIMM flags\n",
2615 nvdimm_name(nvdimm));
2616 return rc;
2617 }
2618
2619 if (nvdimm_has_flush(nfit_blk->nd_region) < 0)
2620 dev_warn(dev, "unable to guarantee persistence of writes\n");
2621
2622 if (mmio->line_size == 0)
2623 return 0;
2624
2625 if ((u32) nfit_blk->cmd_offset % mmio->line_size
2626 + 8 > mmio->line_size) {
2627 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
2628 return -ENXIO;
2629 } else if ((u32) nfit_blk->stat_offset % mmio->line_size
2630 + 8 > mmio->line_size) {
2631 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
2632 return -ENXIO;
2633 }
2634
2635 return 0;
2636 }
2637
2638 static int ars_get_cap(struct acpi_nfit_desc *acpi_desc,
2639 struct nd_cmd_ars_cap *cmd, struct nfit_spa *nfit_spa)
2640 {
2641 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2642 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2643 int cmd_rc, rc;
2644
2645 cmd->address = spa->address;
2646 cmd->length = spa->length;
2647 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, cmd,
2648 sizeof(*cmd), &cmd_rc);
2649 if (rc < 0)
2650 return rc;
2651 return cmd_rc;
2652 }
2653
2654 static int ars_start(struct acpi_nfit_desc *acpi_desc,
2655 struct nfit_spa *nfit_spa, enum nfit_ars_state req_type)
2656 {
2657 int rc;
2658 int cmd_rc;
2659 struct nd_cmd_ars_start ars_start;
2660 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2661 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2662
2663 memset(&ars_start, 0, sizeof(ars_start));
2664 ars_start.address = spa->address;
2665 ars_start.length = spa->length;
2666 if (req_type == ARS_REQ_SHORT)
2667 ars_start.flags = ND_ARS_RETURN_PREV_DATA;
2668 if (nfit_spa_type(spa) == NFIT_SPA_PM)
2669 ars_start.type = ND_ARS_PERSISTENT;
2670 else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE)
2671 ars_start.type = ND_ARS_VOLATILE;
2672 else
2673 return -ENOTTY;
2674
2675 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2676 sizeof(ars_start), &cmd_rc);
2677
2678 if (rc < 0)
2679 return rc;
2680 if (cmd_rc < 0)
2681 return cmd_rc;
2682 set_bit(ARS_VALID, &acpi_desc->scrub_flags);
2683 return 0;
2684 }
2685
2686 static int ars_continue(struct acpi_nfit_desc *acpi_desc)
2687 {
2688 int rc, cmd_rc;
2689 struct nd_cmd_ars_start ars_start;
2690 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2691 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2692
2693 ars_start = (struct nd_cmd_ars_start) {
2694 .address = ars_status->restart_address,
2695 .length = ars_status->restart_length,
2696 .type = ars_status->type,
2697 };
2698 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_START, &ars_start,
2699 sizeof(ars_start), &cmd_rc);
2700 if (rc < 0)
2701 return rc;
2702 return cmd_rc;
2703 }
2704
2705 static int ars_get_status(struct acpi_nfit_desc *acpi_desc)
2706 {
2707 struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
2708 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2709 int rc, cmd_rc;
2710
2711 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_STATUS, ars_status,
2712 acpi_desc->max_ars, &cmd_rc);
2713 if (rc < 0)
2714 return rc;
2715 return cmd_rc;
2716 }
2717
2718 static void ars_complete(struct acpi_nfit_desc *acpi_desc,
2719 struct nfit_spa *nfit_spa)
2720 {
2721 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2722 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2723 struct nd_region *nd_region = nfit_spa->nd_region;
2724 struct device *dev;
2725
2726 lockdep_assert_held(&acpi_desc->init_mutex);
2727 /*
2728 * Only advance the ARS state for ARS runs initiated by the
2729 * kernel, ignore ARS results from BIOS initiated runs for scrub
2730 * completion tracking.
2731 */
2732 if (acpi_desc->scrub_spa != nfit_spa)
2733 return;
2734
2735 if ((ars_status->address >= spa->address && ars_status->address
2736 < spa->address + spa->length)
2737 || (ars_status->address < spa->address)) {
2738 /*
2739 * Assume that if a scrub starts at an offset from the
2740 * start of nfit_spa that we are in the continuation
2741 * case.
2742 *
2743 * Otherwise, if the scrub covers the spa range, mark
2744 * any pending request complete.
2745 */
2746 if (ars_status->address + ars_status->length
2747 >= spa->address + spa->length)
2748 /* complete */;
2749 else
2750 return;
2751 } else
2752 return;
2753
2754 acpi_desc->scrub_spa = NULL;
2755 if (nd_region) {
2756 dev = nd_region_dev(nd_region);
2757 nvdimm_region_notify(nd_region, NVDIMM_REVALIDATE_POISON);
2758 } else
2759 dev = acpi_desc->dev;
2760 dev_dbg(dev, "ARS: range %d complete\n", spa->range_index);
2761 }
2762
2763 static int ars_status_process_records(struct acpi_nfit_desc *acpi_desc)
2764 {
2765 struct nvdimm_bus *nvdimm_bus = acpi_desc->nvdimm_bus;
2766 struct nd_cmd_ars_status *ars_status = acpi_desc->ars_status;
2767 int rc;
2768 u32 i;
2769
2770 /*
2771 * First record starts at 44 byte offset from the start of the
2772 * payload.
2773 */
2774 if (ars_status->out_length < 44)
2775 return 0;
2776
2777 /*
2778 * Ignore potentially stale results that are only refreshed
2779 * after a start-ARS event.
2780 */
2781 if (!test_and_clear_bit(ARS_VALID, &acpi_desc->scrub_flags)) {
2782 dev_dbg(acpi_desc->dev, "skip %d stale records\n",
2783 ars_status->num_records);
2784 return 0;
2785 }
2786
2787 for (i = 0; i < ars_status->num_records; i++) {
2788 /* only process full records */
2789 if (ars_status->out_length
2790 < 44 + sizeof(struct nd_ars_record) * (i + 1))
2791 break;
2792 rc = nvdimm_bus_add_badrange(nvdimm_bus,
2793 ars_status->records[i].err_address,
2794 ars_status->records[i].length);
2795 if (rc)
2796 return rc;
2797 }
2798 if (i < ars_status->num_records)
2799 dev_warn(acpi_desc->dev, "detected truncated ars results\n");
2800
2801 return 0;
2802 }
2803
2804 static void acpi_nfit_remove_resource(void *data)
2805 {
2806 struct resource *res = data;
2807
2808 remove_resource(res);
2809 }
2810
2811 static int acpi_nfit_insert_resource(struct acpi_nfit_desc *acpi_desc,
2812 struct nd_region_desc *ndr_desc)
2813 {
2814 struct resource *res, *nd_res = ndr_desc->res;
2815 int is_pmem, ret;
2816
2817 /* No operation if the region is already registered as PMEM */
2818 is_pmem = region_intersects(nd_res->start, resource_size(nd_res),
2819 IORESOURCE_MEM, IORES_DESC_PERSISTENT_MEMORY);
2820 if (is_pmem == REGION_INTERSECTS)
2821 return 0;
2822
2823 res = devm_kzalloc(acpi_desc->dev, sizeof(*res), GFP_KERNEL);
2824 if (!res)
2825 return -ENOMEM;
2826
2827 res->name = "Persistent Memory";
2828 res->start = nd_res->start;
2829 res->end = nd_res->end;
2830 res->flags = IORESOURCE_MEM;
2831 res->desc = IORES_DESC_PERSISTENT_MEMORY;
2832
2833 ret = insert_resource(&iomem_resource, res);
2834 if (ret)
2835 return ret;
2836
2837 ret = devm_add_action_or_reset(acpi_desc->dev,
2838 acpi_nfit_remove_resource,
2839 res);
2840 if (ret)
2841 return ret;
2842
2843 return 0;
2844 }
2845
2846 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
2847 struct nd_mapping_desc *mapping, struct nd_region_desc *ndr_desc,
2848 struct acpi_nfit_memory_map *memdev,
2849 struct nfit_spa *nfit_spa)
2850 {
2851 struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
2852 memdev->device_handle);
2853 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2854 struct nd_blk_region_desc *ndbr_desc;
2855 struct nfit_mem *nfit_mem;
2856 int rc;
2857
2858 if (!nvdimm) {
2859 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
2860 spa->range_index, memdev->device_handle);
2861 return -ENODEV;
2862 }
2863
2864 mapping->nvdimm = nvdimm;
2865 switch (nfit_spa_type(spa)) {
2866 case NFIT_SPA_PM:
2867 case NFIT_SPA_VOLATILE:
2868 mapping->start = memdev->address;
2869 mapping->size = memdev->region_size;
2870 break;
2871 case NFIT_SPA_DCR:
2872 nfit_mem = nvdimm_provider_data(nvdimm);
2873 if (!nfit_mem || !nfit_mem->bdw) {
2874 dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
2875 spa->range_index, nvdimm_name(nvdimm));
2876 break;
2877 }
2878
2879 mapping->size = nfit_mem->bdw->capacity;
2880 mapping->start = nfit_mem->bdw->start_address;
2881 ndr_desc->num_lanes = nfit_mem->bdw->windows;
2882 ndr_desc->mapping = mapping;
2883 ndr_desc->num_mappings = 1;
2884 ndbr_desc = to_blk_region_desc(ndr_desc);
2885 ndbr_desc->enable = acpi_nfit_blk_region_enable;
2886 ndbr_desc->do_io = acpi_desc->blk_do_io;
2887 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2888 if (rc)
2889 return rc;
2890 nfit_spa->nd_region = nvdimm_blk_region_create(acpi_desc->nvdimm_bus,
2891 ndr_desc);
2892 if (!nfit_spa->nd_region)
2893 return -ENOMEM;
2894 break;
2895 }
2896
2897 return 0;
2898 }
2899
2900 static bool nfit_spa_is_virtual(struct acpi_nfit_system_address *spa)
2901 {
2902 return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2903 nfit_spa_type(spa) == NFIT_SPA_VCD ||
2904 nfit_spa_type(spa) == NFIT_SPA_PDISK ||
2905 nfit_spa_type(spa) == NFIT_SPA_PCD);
2906 }
2907
2908 static bool nfit_spa_is_volatile(struct acpi_nfit_system_address *spa)
2909 {
2910 return (nfit_spa_type(spa) == NFIT_SPA_VDISK ||
2911 nfit_spa_type(spa) == NFIT_SPA_VCD ||
2912 nfit_spa_type(spa) == NFIT_SPA_VOLATILE);
2913 }
2914
2915 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
2916 struct nfit_spa *nfit_spa)
2917 {
2918 static struct nd_mapping_desc mappings[ND_MAX_MAPPINGS];
2919 struct acpi_nfit_system_address *spa = nfit_spa->spa;
2920 struct nd_blk_region_desc ndbr_desc;
2921 struct nd_region_desc *ndr_desc;
2922 struct nfit_memdev *nfit_memdev;
2923 struct nvdimm_bus *nvdimm_bus;
2924 struct resource res;
2925 int count = 0, rc;
2926
2927 if (nfit_spa->nd_region)
2928 return 0;
2929
2930 if (spa->range_index == 0 && !nfit_spa_is_virtual(spa)) {
2931 dev_dbg(acpi_desc->dev, "detected invalid spa index\n");
2932 return 0;
2933 }
2934
2935 memset(&res, 0, sizeof(res));
2936 memset(&mappings, 0, sizeof(mappings));
2937 memset(&ndbr_desc, 0, sizeof(ndbr_desc));
2938 res.start = spa->address;
2939 res.end = res.start + spa->length - 1;
2940 ndr_desc = &ndbr_desc.ndr_desc;
2941 ndr_desc->res = &res;
2942 ndr_desc->provider_data = nfit_spa;
2943 ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
2944 if (spa->flags & ACPI_NFIT_PROXIMITY_VALID) {
2945 ndr_desc->numa_node = acpi_map_pxm_to_online_node(
2946 spa->proximity_domain);
2947 ndr_desc->target_node = acpi_map_pxm_to_node(
2948 spa->proximity_domain);
2949 } else {
2950 ndr_desc->numa_node = NUMA_NO_NODE;
2951 ndr_desc->target_node = NUMA_NO_NODE;
2952 }
2953
2954 /*
2955 * Persistence domain bits are hierarchical, if
2956 * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then
2957 * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied.
2958 */
2959 if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH)
2960 set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags);
2961 else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH)
2962 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags);
2963
2964 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
2965 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
2966 struct nd_mapping_desc *mapping;
2967
2968 if (memdev->range_index != spa->range_index)
2969 continue;
2970 if (count >= ND_MAX_MAPPINGS) {
2971 dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
2972 spa->range_index, ND_MAX_MAPPINGS);
2973 return -ENXIO;
2974 }
2975 mapping = &mappings[count++];
2976 rc = acpi_nfit_init_mapping(acpi_desc, mapping, ndr_desc,
2977 memdev, nfit_spa);
2978 if (rc)
2979 goto out;
2980 }
2981
2982 ndr_desc->mapping = mappings;
2983 ndr_desc->num_mappings = count;
2984 rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
2985 if (rc)
2986 goto out;
2987
2988 nvdimm_bus = acpi_desc->nvdimm_bus;
2989 if (nfit_spa_type(spa) == NFIT_SPA_PM) {
2990 rc = acpi_nfit_insert_resource(acpi_desc, ndr_desc);
2991 if (rc) {
2992 dev_warn(acpi_desc->dev,
2993 "failed to insert pmem resource to iomem: %d\n",
2994 rc);
2995 goto out;
2996 }
2997
2998 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
2999 ndr_desc);
3000 if (!nfit_spa->nd_region)
3001 rc = -ENOMEM;
3002 } else if (nfit_spa_is_volatile(spa)) {
3003 nfit_spa->nd_region = nvdimm_volatile_region_create(nvdimm_bus,
3004 ndr_desc);
3005 if (!nfit_spa->nd_region)
3006 rc = -ENOMEM;
3007 } else if (nfit_spa_is_virtual(spa)) {
3008 nfit_spa->nd_region = nvdimm_pmem_region_create(nvdimm_bus,
3009 ndr_desc);
3010 if (!nfit_spa->nd_region)
3011 rc = -ENOMEM;
3012 }
3013
3014 out:
3015 if (rc)
3016 dev_err(acpi_desc->dev, "failed to register spa range %d\n",
3017 nfit_spa->spa->range_index);
3018 return rc;
3019 }
3020
3021 static int ars_status_alloc(struct acpi_nfit_desc *acpi_desc)
3022 {
3023 struct device *dev = acpi_desc->dev;
3024 struct nd_cmd_ars_status *ars_status;
3025
3026 if (acpi_desc->ars_status) {
3027 memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3028 return 0;
3029 }
3030
3031 ars_status = devm_kzalloc(dev, acpi_desc->max_ars, GFP_KERNEL);
3032 if (!ars_status)
3033 return -ENOMEM;
3034 acpi_desc->ars_status = ars_status;
3035 return 0;
3036 }
3037
3038 static int acpi_nfit_query_poison(struct acpi_nfit_desc *acpi_desc)
3039 {
3040 int rc;
3041
3042 if (ars_status_alloc(acpi_desc))
3043 return -ENOMEM;
3044
3045 rc = ars_get_status(acpi_desc);
3046
3047 if (rc < 0 && rc != -ENOSPC)
3048 return rc;
3049
3050 if (ars_status_process_records(acpi_desc))
3051 dev_err(acpi_desc->dev, "Failed to process ARS records\n");
3052
3053 return rc;
3054 }
3055
3056 static int ars_register(struct acpi_nfit_desc *acpi_desc,
3057 struct nfit_spa *nfit_spa)
3058 {
3059 int rc;
3060
3061 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3062 return acpi_nfit_register_region(acpi_desc, nfit_spa);
3063
3064 set_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3065 if (!no_init_ars)
3066 set_bit(ARS_REQ_LONG, &nfit_spa->ars_state);
3067
3068 switch (acpi_nfit_query_poison(acpi_desc)) {
3069 case 0:
3070 case -ENOSPC:
3071 case -EAGAIN:
3072 rc = ars_start(acpi_desc, nfit_spa, ARS_REQ_SHORT);
3073 /* shouldn't happen, try again later */
3074 if (rc == -EBUSY)
3075 break;
3076 if (rc) {
3077 set_bit(ARS_FAILED, &nfit_spa->ars_state);
3078 break;
3079 }
3080 clear_bit(ARS_REQ_SHORT, &nfit_spa->ars_state);
3081 rc = acpi_nfit_query_poison(acpi_desc);
3082 if (rc)
3083 break;
3084 acpi_desc->scrub_spa = nfit_spa;
3085 ars_complete(acpi_desc, nfit_spa);
3086 /*
3087 * If ars_complete() says we didn't complete the
3088 * short scrub, we'll try again with a long
3089 * request.
3090 */
3091 acpi_desc->scrub_spa = NULL;
3092 break;
3093 case -EBUSY:
3094 case -ENOMEM:
3095 /*
3096 * BIOS was using ARS, wait for it to complete (or
3097 * resources to become available) and then perform our
3098 * own scrubs.
3099 */
3100 break;
3101 default:
3102 set_bit(ARS_FAILED, &nfit_spa->ars_state);
3103 break;
3104 }
3105
3106 return acpi_nfit_register_region(acpi_desc, nfit_spa);
3107 }
3108
3109 static void ars_complete_all(struct acpi_nfit_desc *acpi_desc)
3110 {
3111 struct nfit_spa *nfit_spa;
3112
3113 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3114 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3115 continue;
3116 ars_complete(acpi_desc, nfit_spa);
3117 }
3118 }
3119
3120 static unsigned int __acpi_nfit_scrub(struct acpi_nfit_desc *acpi_desc,
3121 int query_rc)
3122 {
3123 unsigned int tmo = acpi_desc->scrub_tmo;
3124 struct device *dev = acpi_desc->dev;
3125 struct nfit_spa *nfit_spa;
3126
3127 lockdep_assert_held(&acpi_desc->init_mutex);
3128
3129 if (test_bit(ARS_CANCEL, &acpi_desc->scrub_flags))
3130 return 0;
3131
3132 if (query_rc == -EBUSY) {
3133 dev_dbg(dev, "ARS: ARS busy\n");
3134 return min(30U * 60U, tmo * 2);
3135 }
3136 if (query_rc == -ENOSPC) {
3137 dev_dbg(dev, "ARS: ARS continue\n");
3138 ars_continue(acpi_desc);
3139 return 1;
3140 }
3141 if (query_rc && query_rc != -EAGAIN) {
3142 unsigned long long addr, end;
3143
3144 addr = acpi_desc->ars_status->address;
3145 end = addr + acpi_desc->ars_status->length;
3146 dev_dbg(dev, "ARS: %llx-%llx failed (%d)\n", addr, end,
3147 query_rc);
3148 }
3149
3150 ars_complete_all(acpi_desc);
3151 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3152 enum nfit_ars_state req_type;
3153 int rc;
3154
3155 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3156 continue;
3157
3158 /* prefer short ARS requests first */
3159 if (test_bit(ARS_REQ_SHORT, &nfit_spa->ars_state))
3160 req_type = ARS_REQ_SHORT;
3161 else if (test_bit(ARS_REQ_LONG, &nfit_spa->ars_state))
3162 req_type = ARS_REQ_LONG;
3163 else
3164 continue;
3165 rc = ars_start(acpi_desc, nfit_spa, req_type);
3166
3167 dev = nd_region_dev(nfit_spa->nd_region);
3168 dev_dbg(dev, "ARS: range %d ARS start %s (%d)\n",
3169 nfit_spa->spa->range_index,
3170 req_type == ARS_REQ_SHORT ? "short" : "long",
3171 rc);
3172 /*
3173 * Hmm, we raced someone else starting ARS? Try again in
3174 * a bit.
3175 */
3176 if (rc == -EBUSY)
3177 return 1;
3178 if (rc == 0) {
3179 dev_WARN_ONCE(dev, acpi_desc->scrub_spa,
3180 "scrub start while range %d active\n",
3181 acpi_desc->scrub_spa->spa->range_index);
3182 clear_bit(req_type, &nfit_spa->ars_state);
3183 acpi_desc->scrub_spa = nfit_spa;
3184 /*
3185 * Consider this spa last for future scrub
3186 * requests
3187 */
3188 list_move_tail(&nfit_spa->list, &acpi_desc->spas);
3189 return 1;
3190 }
3191
3192 dev_err(dev, "ARS: range %d ARS failed (%d)\n",
3193 nfit_spa->spa->range_index, rc);
3194 set_bit(ARS_FAILED, &nfit_spa->ars_state);
3195 }
3196 return 0;
3197 }
3198
3199 static void __sched_ars(struct acpi_nfit_desc *acpi_desc, unsigned int tmo)
3200 {
3201 lockdep_assert_held(&acpi_desc->init_mutex);
3202
3203 set_bit(ARS_BUSY, &acpi_desc->scrub_flags);
3204 /* note this should only be set from within the workqueue */
3205 if (tmo)
3206 acpi_desc->scrub_tmo = tmo;
3207 queue_delayed_work(nfit_wq, &acpi_desc->dwork, tmo * HZ);
3208 }
3209
3210 static void sched_ars(struct acpi_nfit_desc *acpi_desc)
3211 {
3212 __sched_ars(acpi_desc, 0);
3213 }
3214
3215 static void notify_ars_done(struct acpi_nfit_desc *acpi_desc)
3216 {
3217 lockdep_assert_held(&acpi_desc->init_mutex);
3218
3219 clear_bit(ARS_BUSY, &acpi_desc->scrub_flags);
3220 acpi_desc->scrub_count++;
3221 if (acpi_desc->scrub_count_state)
3222 sysfs_notify_dirent(acpi_desc->scrub_count_state);
3223 }
3224
3225 static void acpi_nfit_scrub(struct work_struct *work)
3226 {
3227 struct acpi_nfit_desc *acpi_desc;
3228 unsigned int tmo;
3229 int query_rc;
3230
3231 acpi_desc = container_of(work, typeof(*acpi_desc), dwork.work);
3232 mutex_lock(&acpi_desc->init_mutex);
3233 query_rc = acpi_nfit_query_poison(acpi_desc);
3234 tmo = __acpi_nfit_scrub(acpi_desc, query_rc);
3235 if (tmo)
3236 __sched_ars(acpi_desc, tmo);
3237 else
3238 notify_ars_done(acpi_desc);
3239 memset(acpi_desc->ars_status, 0, acpi_desc->max_ars);
3240 clear_bit(ARS_POLL, &acpi_desc->scrub_flags);
3241 mutex_unlock(&acpi_desc->init_mutex);
3242 }
3243
3244 static void acpi_nfit_init_ars(struct acpi_nfit_desc *acpi_desc,
3245 struct nfit_spa *nfit_spa)
3246 {
3247 int type = nfit_spa_type(nfit_spa->spa);
3248 struct nd_cmd_ars_cap ars_cap;
3249 int rc;
3250
3251 set_bit(ARS_FAILED, &nfit_spa->ars_state);
3252 memset(&ars_cap, 0, sizeof(ars_cap));
3253 rc = ars_get_cap(acpi_desc, &ars_cap, nfit_spa);
3254 if (rc < 0)
3255 return;
3256 /* check that the supported scrub types match the spa type */
3257 if (type == NFIT_SPA_VOLATILE && ((ars_cap.status >> 16)
3258 & ND_ARS_VOLATILE) == 0)
3259 return;
3260 if (type == NFIT_SPA_PM && ((ars_cap.status >> 16)
3261 & ND_ARS_PERSISTENT) == 0)
3262 return;
3263
3264 nfit_spa->max_ars = ars_cap.max_ars_out;
3265 nfit_spa->clear_err_unit = ars_cap.clear_err_unit;
3266 acpi_desc->max_ars = max(nfit_spa->max_ars, acpi_desc->max_ars);
3267 clear_bit(ARS_FAILED, &nfit_spa->ars_state);
3268 }
3269
3270 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
3271 {
3272 struct nfit_spa *nfit_spa;
3273 int rc;
3274
3275 set_bit(ARS_VALID, &acpi_desc->scrub_flags);
3276 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3277 switch (nfit_spa_type(nfit_spa->spa)) {
3278 case NFIT_SPA_VOLATILE:
3279 case NFIT_SPA_PM:
3280 acpi_nfit_init_ars(acpi_desc, nfit_spa);
3281 break;
3282 }
3283 }
3284
3285 list_for_each_entry(nfit_spa, &acpi_desc->spas, list)
3286 switch (nfit_spa_type(nfit_spa->spa)) {
3287 case NFIT_SPA_VOLATILE:
3288 case NFIT_SPA_PM:
3289 /* register regions and kick off initial ARS run */
3290 rc = ars_register(acpi_desc, nfit_spa);
3291 if (rc)
3292 return rc;
3293 break;
3294 case NFIT_SPA_BDW:
3295 /* nothing to register */
3296 break;
3297 case NFIT_SPA_DCR:
3298 case NFIT_SPA_VDISK:
3299 case NFIT_SPA_VCD:
3300 case NFIT_SPA_PDISK:
3301 case NFIT_SPA_PCD:
3302 /* register known regions that don't support ARS */
3303 rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
3304 if (rc)
3305 return rc;
3306 break;
3307 default:
3308 /* don't register unknown regions */
3309 break;
3310 }
3311
3312 sched_ars(acpi_desc);
3313 return 0;
3314 }
3315
3316 static int acpi_nfit_check_deletions(struct acpi_nfit_desc *acpi_desc,
3317 struct nfit_table_prev *prev)
3318 {
3319 struct device *dev = acpi_desc->dev;
3320
3321 if (!list_empty(&prev->spas) ||
3322 !list_empty(&prev->memdevs) ||
3323 !list_empty(&prev->dcrs) ||
3324 !list_empty(&prev->bdws) ||
3325 !list_empty(&prev->idts) ||
3326 !list_empty(&prev->flushes)) {
3327 dev_err(dev, "new nfit deletes entries (unsupported)\n");
3328 return -ENXIO;
3329 }
3330 return 0;
3331 }
3332
3333 static int acpi_nfit_desc_init_scrub_attr(struct acpi_nfit_desc *acpi_desc)
3334 {
3335 struct device *dev = acpi_desc->dev;
3336 struct kernfs_node *nfit;
3337 struct device *bus_dev;
3338
3339 if (!ars_supported(acpi_desc->nvdimm_bus))
3340 return 0;
3341
3342 bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3343 nfit = sysfs_get_dirent(bus_dev->kobj.sd, "nfit");
3344 if (!nfit) {
3345 dev_err(dev, "sysfs_get_dirent 'nfit' failed\n");
3346 return -ENODEV;
3347 }
3348 acpi_desc->scrub_count_state = sysfs_get_dirent(nfit, "scrub");
3349 sysfs_put(nfit);
3350 if (!acpi_desc->scrub_count_state) {
3351 dev_err(dev, "sysfs_get_dirent 'scrub' failed\n");
3352 return -ENODEV;
3353 }
3354
3355 return 0;
3356 }
3357
3358 static void acpi_nfit_unregister(void *data)
3359 {
3360 struct acpi_nfit_desc *acpi_desc = data;
3361
3362 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
3363 }
3364
3365 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, void *data, acpi_size sz)
3366 {
3367 struct device *dev = acpi_desc->dev;
3368 struct nfit_table_prev prev;
3369 const void *end;
3370 int rc;
3371
3372 if (!acpi_desc->nvdimm_bus) {
3373 acpi_nfit_init_dsms(acpi_desc);
3374
3375 acpi_desc->nvdimm_bus = nvdimm_bus_register(dev,
3376 &acpi_desc->nd_desc);
3377 if (!acpi_desc->nvdimm_bus)
3378 return -ENOMEM;
3379
3380 rc = devm_add_action_or_reset(dev, acpi_nfit_unregister,
3381 acpi_desc);
3382 if (rc)
3383 return rc;
3384
3385 rc = acpi_nfit_desc_init_scrub_attr(acpi_desc);
3386 if (rc)
3387 return rc;
3388
3389 /* register this acpi_desc for mce notifications */
3390 mutex_lock(&acpi_desc_lock);
3391 list_add_tail(&acpi_desc->list, &acpi_descs);
3392 mutex_unlock(&acpi_desc_lock);
3393 }
3394
3395 mutex_lock(&acpi_desc->init_mutex);
3396
3397 INIT_LIST_HEAD(&prev.spas);
3398 INIT_LIST_HEAD(&prev.memdevs);
3399 INIT_LIST_HEAD(&prev.dcrs);
3400 INIT_LIST_HEAD(&prev.bdws);
3401 INIT_LIST_HEAD(&prev.idts);
3402 INIT_LIST_HEAD(&prev.flushes);
3403
3404 list_cut_position(&prev.spas, &acpi_desc->spas,
3405 acpi_desc->spas.prev);
3406 list_cut_position(&prev.memdevs, &acpi_desc->memdevs,
3407 acpi_desc->memdevs.prev);
3408 list_cut_position(&prev.dcrs, &acpi_desc->dcrs,
3409 acpi_desc->dcrs.prev);
3410 list_cut_position(&prev.bdws, &acpi_desc->bdws,
3411 acpi_desc->bdws.prev);
3412 list_cut_position(&prev.idts, &acpi_desc->idts,
3413 acpi_desc->idts.prev);
3414 list_cut_position(&prev.flushes, &acpi_desc->flushes,
3415 acpi_desc->flushes.prev);
3416
3417 end = data + sz;
3418 while (!IS_ERR_OR_NULL(data))
3419 data = add_table(acpi_desc, &prev, data, end);
3420
3421 if (IS_ERR(data)) {
3422 dev_dbg(dev, "nfit table parsing error: %ld\n", PTR_ERR(data));
3423 rc = PTR_ERR(data);
3424 goto out_unlock;
3425 }
3426
3427 rc = acpi_nfit_check_deletions(acpi_desc, &prev);
3428 if (rc)
3429 goto out_unlock;
3430
3431 rc = nfit_mem_init(acpi_desc);
3432 if (rc)
3433 goto out_unlock;
3434
3435 rc = acpi_nfit_register_dimms(acpi_desc);
3436 if (rc)
3437 goto out_unlock;
3438
3439 rc = acpi_nfit_register_regions(acpi_desc);
3440
3441 out_unlock:
3442 mutex_unlock(&acpi_desc->init_mutex);
3443 return rc;
3444 }
3445 EXPORT_SYMBOL_GPL(acpi_nfit_init);
3446
3447 static int acpi_nfit_flush_probe(struct nvdimm_bus_descriptor *nd_desc)
3448 {
3449 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3450 struct device *dev = acpi_desc->dev;
3451
3452 /* Bounce the device lock to flush acpi_nfit_add / acpi_nfit_notify */
3453 nfit_device_lock(dev);
3454 nfit_device_unlock(dev);
3455
3456 /* Bounce the init_mutex to complete initial registration */
3457 mutex_lock(&acpi_desc->init_mutex);
3458 mutex_unlock(&acpi_desc->init_mutex);
3459
3460 return 0;
3461 }
3462
3463 static int __acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3464 struct nvdimm *nvdimm, unsigned int cmd)
3465 {
3466 struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
3467
3468 if (nvdimm)
3469 return 0;
3470 if (cmd != ND_CMD_ARS_START)
3471 return 0;
3472
3473 /*
3474 * The kernel and userspace may race to initiate a scrub, but
3475 * the scrub thread is prepared to lose that initial race. It
3476 * just needs guarantees that any ARS it initiates are not
3477 * interrupted by any intervening start requests from userspace.
3478 */
3479 if (work_busy(&acpi_desc->dwork.work))
3480 return -EBUSY;
3481
3482 return 0;
3483 }
3484
3485 /* prevent security commands from being issued via ioctl */
3486 static int acpi_nfit_clear_to_send(struct nvdimm_bus_descriptor *nd_desc,
3487 struct nvdimm *nvdimm, unsigned int cmd, void *buf)
3488 {
3489 struct nd_cmd_pkg *call_pkg = buf;
3490 unsigned int func;
3491
3492 if (nvdimm && cmd == ND_CMD_CALL &&
3493 call_pkg->nd_family == NVDIMM_FAMILY_INTEL) {
3494 func = call_pkg->nd_command;
3495 if ((1 << func) & NVDIMM_INTEL_SECURITY_CMDMASK)
3496 return -EOPNOTSUPP;
3497 }
3498
3499 return __acpi_nfit_clear_to_send(nd_desc, nvdimm, cmd);
3500 }
3501
3502 int acpi_nfit_ars_rescan(struct acpi_nfit_desc *acpi_desc,
3503 enum nfit_ars_state req_type)
3504 {
3505 struct device *dev = acpi_desc->dev;
3506 int scheduled = 0, busy = 0;
3507 struct nfit_spa *nfit_spa;
3508
3509 mutex_lock(&acpi_desc->init_mutex);
3510 if (test_bit(ARS_CANCEL, &acpi_desc->scrub_flags)) {
3511 mutex_unlock(&acpi_desc->init_mutex);
3512 return 0;
3513 }
3514
3515 list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
3516 int type = nfit_spa_type(nfit_spa->spa);
3517
3518 if (type != NFIT_SPA_PM && type != NFIT_SPA_VOLATILE)
3519 continue;
3520 if (test_bit(ARS_FAILED, &nfit_spa->ars_state))
3521 continue;
3522
3523 if (test_and_set_bit(req_type, &nfit_spa->ars_state))
3524 busy++;
3525 else
3526 scheduled++;
3527 }
3528 if (scheduled) {
3529 sched_ars(acpi_desc);
3530 dev_dbg(dev, "ars_scan triggered\n");
3531 }
3532 mutex_unlock(&acpi_desc->init_mutex);
3533
3534 if (scheduled)
3535 return 0;
3536 if (busy)
3537 return -EBUSY;
3538 return -ENOTTY;
3539 }
3540
3541 void acpi_nfit_desc_init(struct acpi_nfit_desc *acpi_desc, struct device *dev)
3542 {
3543 struct nvdimm_bus_descriptor *nd_desc;
3544
3545 dev_set_drvdata(dev, acpi_desc);
3546 acpi_desc->dev = dev;
3547 acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
3548 nd_desc = &acpi_desc->nd_desc;
3549 nd_desc->provider_name = "ACPI.NFIT";
3550 nd_desc->module = THIS_MODULE;
3551 nd_desc->ndctl = acpi_nfit_ctl;
3552 nd_desc->flush_probe = acpi_nfit_flush_probe;
3553 nd_desc->clear_to_send = acpi_nfit_clear_to_send;
3554 nd_desc->attr_groups = acpi_nfit_attribute_groups;
3555
3556 INIT_LIST_HEAD(&acpi_desc->spas);
3557 INIT_LIST_HEAD(&acpi_desc->dcrs);
3558 INIT_LIST_HEAD(&acpi_desc->bdws);
3559 INIT_LIST_HEAD(&acpi_desc->idts);
3560 INIT_LIST_HEAD(&acpi_desc->flushes);
3561 INIT_LIST_HEAD(&acpi_desc->memdevs);
3562 INIT_LIST_HEAD(&acpi_desc->dimms);
3563 INIT_LIST_HEAD(&acpi_desc->list);
3564 mutex_init(&acpi_desc->init_mutex);
3565 acpi_desc->scrub_tmo = 1;
3566 INIT_DELAYED_WORK(&acpi_desc->dwork, acpi_nfit_scrub);
3567 }
3568 EXPORT_SYMBOL_GPL(acpi_nfit_desc_init);
3569
3570 static void acpi_nfit_put_table(void *table)
3571 {
3572 acpi_put_table(table);
3573 }
3574
3575 void acpi_nfit_shutdown(void *data)
3576 {
3577 struct acpi_nfit_desc *acpi_desc = data;
3578 struct device *bus_dev = to_nvdimm_bus_dev(acpi_desc->nvdimm_bus);
3579
3580 /*
3581 * Destruct under acpi_desc_lock so that nfit_handle_mce does not
3582 * race teardown
3583 */
3584 mutex_lock(&acpi_desc_lock);
3585 list_del(&acpi_desc->list);
3586 mutex_unlock(&acpi_desc_lock);
3587
3588 mutex_lock(&acpi_desc->init_mutex);
3589 set_bit(ARS_CANCEL, &acpi_desc->scrub_flags);
3590 cancel_delayed_work_sync(&acpi_desc->dwork);
3591 mutex_unlock(&acpi_desc->init_mutex);
3592
3593 /*
3594 * Bounce the nvdimm bus lock to make sure any in-flight
3595 * acpi_nfit_ars_rescan() submissions have had a chance to
3596 * either submit or see ->cancel set.
3597 */
3598 nfit_device_lock(bus_dev);
3599 nfit_device_unlock(bus_dev);
3600
3601 flush_workqueue(nfit_wq);
3602 }
3603 EXPORT_SYMBOL_GPL(acpi_nfit_shutdown);
3604
3605 static int acpi_nfit_add(struct acpi_device *adev)
3606 {
3607 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3608 struct acpi_nfit_desc *acpi_desc;
3609 struct device *dev = &adev->dev;
3610 struct acpi_table_header *tbl;
3611 acpi_status status = AE_OK;
3612 acpi_size sz;
3613 int rc = 0;
3614
3615 status = acpi_get_table(ACPI_SIG_NFIT, 0, &tbl);
3616 if (ACPI_FAILURE(status)) {
3617 /* The NVDIMM root device allows OS to trigger enumeration of
3618 * NVDIMMs through NFIT at boot time and re-enumeration at
3619 * root level via the _FIT method during runtime.
3620 * This is ok to return 0 here, we could have an nvdimm
3621 * hotplugged later and evaluate _FIT method which returns
3622 * data in the format of a series of NFIT Structures.
3623 */
3624 dev_dbg(dev, "failed to find NFIT at startup\n");
3625 return 0;
3626 }
3627
3628 rc = devm_add_action_or_reset(dev, acpi_nfit_put_table, tbl);
3629 if (rc)
3630 return rc;
3631 sz = tbl->length;
3632
3633 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3634 if (!acpi_desc)
3635 return -ENOMEM;
3636 acpi_nfit_desc_init(acpi_desc, &adev->dev);
3637
3638 /* Save the acpi header for exporting the revision via sysfs */
3639 acpi_desc->acpi_header = *tbl;
3640
3641 /* Evaluate _FIT and override with that if present */
3642 status = acpi_evaluate_object(adev->handle, "_FIT", NULL, &buf);
3643 if (ACPI_SUCCESS(status) && buf.length > 0) {
3644 union acpi_object *obj = buf.pointer;
3645
3646 if (obj->type == ACPI_TYPE_BUFFER)
3647 rc = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3648 obj->buffer.length);
3649 else
3650 dev_dbg(dev, "invalid type %d, ignoring _FIT\n",
3651 (int) obj->type);
3652 kfree(buf.pointer);
3653 } else
3654 /* skip over the lead-in header table */
3655 rc = acpi_nfit_init(acpi_desc, (void *) tbl
3656 + sizeof(struct acpi_table_nfit),
3657 sz - sizeof(struct acpi_table_nfit));
3658
3659 if (rc)
3660 return rc;
3661 return devm_add_action_or_reset(dev, acpi_nfit_shutdown, acpi_desc);
3662 }
3663
3664 static int acpi_nfit_remove(struct acpi_device *adev)
3665 {
3666 /* see acpi_nfit_unregister */
3667 return 0;
3668 }
3669
3670 static void acpi_nfit_update_notify(struct device *dev, acpi_handle handle)
3671 {
3672 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3673 struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
3674 union acpi_object *obj;
3675 acpi_status status;
3676 int ret;
3677
3678 if (!dev->driver) {
3679 /* dev->driver may be null if we're being removed */
3680 dev_dbg(dev, "no driver found for dev\n");
3681 return;
3682 }
3683
3684 if (!acpi_desc) {
3685 acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
3686 if (!acpi_desc)
3687 return;
3688 acpi_nfit_desc_init(acpi_desc, dev);
3689 } else {
3690 /*
3691 * Finish previous registration before considering new
3692 * regions.
3693 */
3694 flush_workqueue(nfit_wq);
3695 }
3696
3697 /* Evaluate _FIT */
3698 status = acpi_evaluate_object(handle, "_FIT", NULL, &buf);
3699 if (ACPI_FAILURE(status)) {
3700 dev_err(dev, "failed to evaluate _FIT\n");
3701 return;
3702 }
3703
3704 obj = buf.pointer;
3705 if (obj->type == ACPI_TYPE_BUFFER) {
3706 ret = acpi_nfit_init(acpi_desc, obj->buffer.pointer,
3707 obj->buffer.length);
3708 if (ret)
3709 dev_err(dev, "failed to merge updated NFIT\n");
3710 } else
3711 dev_err(dev, "Invalid _FIT\n");
3712 kfree(buf.pointer);
3713 }
3714
3715 static void acpi_nfit_uc_error_notify(struct device *dev, acpi_handle handle)
3716 {
3717 struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(dev);
3718
3719 if (acpi_desc->scrub_mode == HW_ERROR_SCRUB_ON)
3720 acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_LONG);
3721 else
3722 acpi_nfit_ars_rescan(acpi_desc, ARS_REQ_SHORT);
3723 }
3724
3725 void __acpi_nfit_notify(struct device *dev, acpi_handle handle, u32 event)
3726 {
3727 dev_dbg(dev, "event: 0x%x\n", event);
3728
3729 switch (event) {
3730 case NFIT_NOTIFY_UPDATE:
3731 return acpi_nfit_update_notify(dev, handle);
3732 case NFIT_NOTIFY_UC_MEMORY_ERROR:
3733 return acpi_nfit_uc_error_notify(dev, handle);
3734 default:
3735 return;
3736 }
3737 }
3738 EXPORT_SYMBOL_GPL(__acpi_nfit_notify);
3739
3740 static void acpi_nfit_notify(struct acpi_device *adev, u32 event)
3741 {
3742 nfit_device_lock(&adev->dev);
3743 __acpi_nfit_notify(&adev->dev, adev->handle, event);
3744 nfit_device_unlock(&adev->dev);
3745 }
3746
3747 static const struct acpi_device_id acpi_nfit_ids[] = {
3748 { "ACPI0012", 0 },
3749 { "", 0 },
3750 };
3751 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
3752
3753 static struct acpi_driver acpi_nfit_driver = {
3754 .name = KBUILD_MODNAME,
3755 .ids = acpi_nfit_ids,
3756 .ops = {
3757 .add = acpi_nfit_add,
3758 .remove = acpi_nfit_remove,
3759 .notify = acpi_nfit_notify,
3760 },
3761 };
3762
3763 static __init int nfit_init(void)
3764 {
3765 int ret;
3766
3767 BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
3768 BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
3769 BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
3770 BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
3771 BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
3772 BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
3773 BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
3774 BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16);
3775
3776 guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]);
3777 guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]);
3778 guid_parse(UUID_CONTROL_REGION, &nfit_uuid[NFIT_SPA_DCR]);
3779 guid_parse(UUID_DATA_REGION, &nfit_uuid[NFIT_SPA_BDW]);
3780 guid_parse(UUID_VOLATILE_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_VDISK]);
3781 guid_parse(UUID_VOLATILE_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_VCD]);
3782 guid_parse(UUID_PERSISTENT_VIRTUAL_DISK, &nfit_uuid[NFIT_SPA_PDISK]);
3783 guid_parse(UUID_PERSISTENT_VIRTUAL_CD, &nfit_uuid[NFIT_SPA_PCD]);
3784 guid_parse(UUID_NFIT_BUS, &nfit_uuid[NFIT_DEV_BUS]);
3785 guid_parse(UUID_NFIT_DIMM, &nfit_uuid[NFIT_DEV_DIMM]);
3786 guid_parse(UUID_NFIT_DIMM_N_HPE1, &nfit_uuid[NFIT_DEV_DIMM_N_HPE1]);
3787 guid_parse(UUID_NFIT_DIMM_N_HPE2, &nfit_uuid[NFIT_DEV_DIMM_N_HPE2]);
3788 guid_parse(UUID_NFIT_DIMM_N_MSFT, &nfit_uuid[NFIT_DEV_DIMM_N_MSFT]);
3789 guid_parse(UUID_NFIT_DIMM_N_HYPERV, &nfit_uuid[NFIT_DEV_DIMM_N_HYPERV]);
3790
3791 nfit_wq = create_singlethread_workqueue("nfit");
3792 if (!nfit_wq)
3793 return -ENOMEM;
3794
3795 nfit_mce_register();
3796 ret = acpi_bus_register_driver(&acpi_nfit_driver);
3797 if (ret) {
3798 nfit_mce_unregister();
3799 destroy_workqueue(nfit_wq);
3800 }
3801
3802 return ret;
3803
3804 }
3805
3806 static __exit void nfit_exit(void)
3807 {
3808 nfit_mce_unregister();
3809 acpi_bus_unregister_driver(&acpi_nfit_driver);
3810 destroy_workqueue(nfit_wq);
3811 WARN_ON(!list_empty(&acpi_descs));
3812 }
3813
3814 module_init(nfit_init);
3815 module_exit(nfit_exit);
3816 MODULE_LICENSE("GPL v2");
3817 MODULE_AUTHOR("Intel Corporation");
Linux with added FPC-III support