?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for-linus-20190706' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / sbus / char / oradax.c
blob8af216287a843c865194617ce4bf9e030ee5dd99
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
4 */
5
6 /*
7 * Oracle Data Analytics Accelerator (DAX)
8 *
9 * DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8
10 * (DAX2) processor chips, and has direct access to the CPU's L3
11 * caches as well as physical memory. It can perform several
12 * operations on data streams with various input and output formats.
13 * The driver provides a transport mechanism only and has limited
14 * knowledge of the various opcodes and data formats. A user space
15 * library provides high level services and translates these into low
16 * level commands which are then passed into the driver and
17 * subsequently the hypervisor and the coprocessor. The library is
18 * the recommended way for applications to use the coprocessor, and
19 * the driver interface is not intended for general use.
20 *
21 * See Documentation/sparc/oradax/oracle-dax.rst for more details.
22 */
23
24 #include <linux/uaccess.h>
25 #include <linux/module.h>
26 #include <linux/delay.h>
27 #include <linux/cdev.h>
28 #include <linux/slab.h>
29 #include <linux/mm.h>
30
31 #include <asm/hypervisor.h>
32 #include <asm/mdesc.h>
33 #include <asm/oradax.h>
34
35 MODULE_LICENSE("GPL");
36 MODULE_DESCRIPTION("Driver for Oracle Data Analytics Accelerator");
37
38 #define DAX_DBG_FLG_BASIC 0x01
39 #define DAX_DBG_FLG_STAT 0x02
40 #define DAX_DBG_FLG_INFO 0x04
41 #define DAX_DBG_FLG_ALL 0xff
42
43 #define dax_err(fmt, ...) pr_err("%s: " fmt "\n", __func__, ##__VA_ARGS__)
44 #define dax_info(fmt, ...) pr_info("%s: " fmt "\n", __func__, ##__VA_ARGS__)
45
46 #define dax_dbg(fmt, ...) do { \
47 if (dax_debug & DAX_DBG_FLG_BASIC)\
48 dax_info(fmt, ##__VA_ARGS__); \
49 } while (0)
50 #define dax_stat_dbg(fmt, ...) do { \
51 if (dax_debug & DAX_DBG_FLG_STAT) \
52 dax_info(fmt, ##__VA_ARGS__); \
53 } while (0)
54 #define dax_info_dbg(fmt, ...) do { \
55 if (dax_debug & DAX_DBG_FLG_INFO) \
56 dax_info(fmt, ##__VA_ARGS__); \
57 } while (0)
58
59 #define DAX1_MINOR 1
60 #define DAX1_MAJOR 1
61 #define DAX2_MINOR 0
62 #define DAX2_MAJOR 2
63
64 #define DAX1_STR "ORCL,sun4v-dax"
65 #define DAX2_STR "ORCL,sun4v-dax2"
66
67 #define DAX_CA_ELEMS (DAX_MMAP_LEN / sizeof(struct dax_cca))
68
69 #define DAX_CCB_USEC 100
70 #define DAX_CCB_RETRIES 10000
71
72 /* stream types */
73 enum {
74 OUT,
75 PRI,
76 SEC,
77 TBL,
78 NUM_STREAM_TYPES
79 };
80
81 /* completion status */
82 #define CCA_STAT_NOT_COMPLETED 0
83 #define CCA_STAT_COMPLETED 1
84 #define CCA_STAT_FAILED 2
85 #define CCA_STAT_KILLED 3
86 #define CCA_STAT_NOT_RUN 4
87 #define CCA_STAT_PIPE_OUT 5
88 #define CCA_STAT_PIPE_SRC 6
89 #define CCA_STAT_PIPE_DST 7
90
91 /* completion err */
92 #define CCA_ERR_SUCCESS 0x0 /* no error */
93 #define CCA_ERR_OVERFLOW 0x1 /* buffer overflow */
94 #define CCA_ERR_DECODE 0x2 /* CCB decode error */
95 #define CCA_ERR_PAGE_OVERFLOW 0x3 /* page overflow */
96 #define CCA_ERR_KILLED 0x7 /* command was killed */
97 #define CCA_ERR_TIMEOUT 0x8 /* Timeout */
98 #define CCA_ERR_ADI 0x9 /* ADI error */
99 #define CCA_ERR_DATA_FMT 0xA /* data format error */
100 #define CCA_ERR_OTHER_NO_RETRY 0xE /* Other error, do not retry */
101 #define CCA_ERR_OTHER_RETRY 0xF /* Other error, retry */
102 #define CCA_ERR_PARTIAL_SYMBOL 0x80 /* QP partial symbol warning */
103
104 /* CCB address types */
105 #define DAX_ADDR_TYPE_NONE 0
106 #define DAX_ADDR_TYPE_VA_ALT 1 /* secondary context */
107 #define DAX_ADDR_TYPE_RA 2 /* real address */
108 #define DAX_ADDR_TYPE_VA 3 /* virtual address */
109
110 /* dax_header_t opcode */
111 #define DAX_OP_SYNC_NOP 0x0
112 #define DAX_OP_EXTRACT 0x1
113 #define DAX_OP_SCAN_VALUE 0x2
114 #define DAX_OP_SCAN_RANGE 0x3
115 #define DAX_OP_TRANSLATE 0x4
116 #define DAX_OP_SELECT 0x5
117 #define DAX_OP_INVERT 0x10 /* OR with translate, scan opcodes */
118
119 struct dax_header {
120 u32 ccb_version:4; /* 31:28 CCB Version */
121 /* 27:24 Sync Flags */
122 u32 pipe:1; /* Pipeline */
123 u32 longccb:1; /* Longccb. Set for scan with lu2, lu3, lu4. */
124 u32 cond:1; /* Conditional */
125 u32 serial:1; /* Serial */
126 u32 opcode:8; /* 23:16 Opcode */
127 /* 15:0 Address Type. */
128 u32 reserved:3; /* 15:13 reserved */
129 u32 table_addr_type:2; /* 12:11 Huffman Table Address Type */
130 u32 out_addr_type:3; /* 10:8 Destination Address Type */
131 u32 sec_addr_type:3; /* 7:5 Secondary Source Address Type */
132 u32 pri_addr_type:3; /* 4:2 Primary Source Address Type */
133 u32 cca_addr_type:2; /* 1:0 Completion Address Type */
134 };
135
136 struct dax_control {
137 u32 pri_fmt:4; /* 31:28 Primary Input Format */
138 u32 pri_elem_size:5; /* 27:23 Primary Input Element Size(less1) */
139 u32 pri_offset:3; /* 22:20 Primary Input Starting Offset */
140 u32 sec_encoding:1; /* 19 Secondary Input Encoding */
141 /* (must be 0 for Select) */
142 u32 sec_offset:3; /* 18:16 Secondary Input Starting Offset */
143 u32 sec_elem_size:2; /* 15:14 Secondary Input Element Size */
144 /* (must be 0 for Select) */
145 u32 out_fmt:2; /* 13:12 Output Format */
146 u32 out_elem_size:2; /* 11:10 Output Element Size */
147 u32 misc:10; /* 9:0 Opcode specific info */
148 };
149
150 struct dax_data_access {
151 u64 flow_ctrl:2; /* 63:62 Flow Control Type */
152 u64 pipe_target:2; /* 61:60 Pipeline Target */
153 u64 out_buf_size:20; /* 59:40 Output Buffer Size */
154 /* (cachelines less 1) */
155 u64 unused1:8; /* 39:32 Reserved, Set to 0 */
156 u64 out_alloc:5; /* 31:27 Output Allocation */
157 u64 unused2:1; /* 26 Reserved */
158 u64 pri_len_fmt:2; /* 25:24 Input Length Format */
159 u64 pri_len:24; /* 23:0 Input Element/Byte/Bit Count */
160 /* (less 1) */
161 };
162
163 struct dax_ccb {
164 struct dax_header hdr; /* CCB Header */
165 struct dax_control ctrl;/* Control Word */
166 void *ca; /* Completion Address */
167 void *pri; /* Primary Input Address */
168 struct dax_data_access dac; /* Data Access Control */
169 void *sec; /* Secondary Input Address */
170 u64 dword5; /* depends on opcode */
171 void *out; /* Output Address */
172 void *tbl; /* Table Address or bitmap */
173 };
174
175 struct dax_cca {
176 u8 status; /* user may mwait on this address */
177 u8 err; /* user visible error notification */
178 u8 rsvd[2]; /* reserved */
179 u32 n_remaining; /* for QP partial symbol warning */
180 u32 output_sz; /* output in bytes */
181 u32 rsvd2; /* reserved */
182 u64 run_cycles; /* run time in OCND2 cycles */
183 u64 run_stats; /* nothing reported in version 1.0 */
184 u32 n_processed; /* number input elements */
185 u32 rsvd3[5]; /* reserved */
186 u64 retval; /* command return value */
187 u64 rsvd4[8]; /* reserved */
188 };
189
190 /* per thread CCB context */
191 struct dax_ctx {
192 struct dax_ccb *ccb_buf;
193 u64 ccb_buf_ra; /* cached RA of ccb_buf */
194 struct dax_cca *ca_buf;
195 u64 ca_buf_ra; /* cached RA of ca_buf */
196 struct page *pages[DAX_CA_ELEMS][NUM_STREAM_TYPES];
197 /* array of locked pages */
198 struct task_struct *owner; /* thread that owns ctx */
199 struct task_struct *client; /* requesting thread */
200 union ccb_result result;
201 u32 ccb_count;
202 u32 fail_count;
203 };
204
205 /* driver public entry points */
206 static int dax_open(struct inode *inode, struct file *file);
207 static ssize_t dax_read(struct file *filp, char __user *buf,
208 size_t count, loff_t *ppos);
209 static ssize_t dax_write(struct file *filp, const char __user *buf,
210 size_t count, loff_t *ppos);
211 static int dax_devmap(struct file *f, struct vm_area_struct *vma);
212 static int dax_close(struct inode *i, struct file *f);
213
214 static const struct file_operations dax_fops = {
215 .owner = THIS_MODULE,
216 .open = dax_open,
217 .read = dax_read,
218 .write = dax_write,
219 .mmap = dax_devmap,
220 .release = dax_close,
221 };
222
223 static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,
224 size_t count, loff_t *ppos);
225 static int dax_ccb_info(u64 ca, struct ccb_info_result *info);
226 static int dax_ccb_kill(u64 ca, u16 *kill_res);
227
228 static struct cdev c_dev;
229 static struct class *cl;
230 static dev_t first;
231
232 static int max_ccb_version;
233 static int dax_debug;
234 module_param(dax_debug, int, 0644);
235 MODULE_PARM_DESC(dax_debug, "Debug flags");
236
237 static int __init dax_attach(void)
238 {
239 unsigned long dummy, hv_rv, major, minor, minor_requested, max_ccbs;
240 struct mdesc_handle *hp = mdesc_grab();
241 char *prop, *dax_name;
242 bool found = false;
243 int len, ret = 0;
244 u64 pn;
245
246 if (hp == NULL) {
247 dax_err("Unable to grab mdesc");
248 return -ENODEV;
249 }
250
251 mdesc_for_each_node_by_name(hp, pn, "virtual-device") {
252 prop = (char *)mdesc_get_property(hp, pn, "name", &len);
253 if (prop == NULL)
254 continue;
255 if (strncmp(prop, "dax", strlen("dax")))
256 continue;
257 dax_dbg("Found node 0x%llx = %s", pn, prop);
258
259 prop = (char *)mdesc_get_property(hp, pn, "compatible", &len);
260 if (prop == NULL)
261 continue;
262 dax_dbg("Found node 0x%llx = %s", pn, prop);
263 found = true;
264 break;
265 }
266
267 if (!found) {
268 dax_err("No DAX device found");
269 ret = -ENODEV;
270 goto done;
271 }
272
273 if (strncmp(prop, DAX2_STR, strlen(DAX2_STR)) == 0) {
274 dax_name = DAX_NAME "2";
275 major = DAX2_MAJOR;
276 minor_requested = DAX2_MINOR;
277 max_ccb_version = 1;
278 dax_dbg("MD indicates DAX2 coprocessor");
279 } else if (strncmp(prop, DAX1_STR, strlen(DAX1_STR)) == 0) {
280 dax_name = DAX_NAME "1";
281 major = DAX1_MAJOR;
282 minor_requested = DAX1_MINOR;
283 max_ccb_version = 0;
284 dax_dbg("MD indicates DAX1 coprocessor");
285 } else {
286 dax_err("Unknown dax type: %s", prop);
287 ret = -ENODEV;
288 goto done;
289 }
290
291 minor = minor_requested;
292 dax_dbg("Registering DAX HV api with major %ld minor %ld", major,
293 minor);
294 if (sun4v_hvapi_register(HV_GRP_DAX, major, &minor)) {
295 dax_err("hvapi_register failed");
296 ret = -ENODEV;
297 goto done;
298 } else {
299 dax_dbg("Max minor supported by HV = %ld (major %ld)", minor,
300 major);
301 minor = min(minor, minor_requested);
302 dax_dbg("registered DAX major %ld minor %ld", major, minor);
303 }
304
305 /* submit a zero length ccb array to query coprocessor queue size */
306 hv_rv = sun4v_ccb_submit(0, 0, HV_CCB_QUERY_CMD, 0, &max_ccbs, &dummy);
307 if (hv_rv != 0) {
308 dax_err("get_hwqueue_size failed with status=%ld and max_ccbs=%ld",
309 hv_rv, max_ccbs);
310 ret = -ENODEV;
311 goto done;
312 }
313
314 if (max_ccbs != DAX_MAX_CCBS) {
315 dax_err("HV reports unsupported max_ccbs=%ld", max_ccbs);
316 ret = -ENODEV;
317 goto done;
318 }
319
320 if (alloc_chrdev_region(&first, 0, 1, DAX_NAME) < 0) {
321 dax_err("alloc_chrdev_region failed");
322 ret = -ENXIO;
323 goto done;
324 }
325
326 cl = class_create(THIS_MODULE, DAX_NAME);
327 if (IS_ERR(cl)) {
328 dax_err("class_create failed");
329 ret = PTR_ERR(cl);
330 goto class_error;
331 }
332
333 if (device_create(cl, NULL, first, NULL, dax_name) == NULL) {
334 dax_err("device_create failed");
335 ret = -ENXIO;
336 goto device_error;
337 }
338
339 cdev_init(&c_dev, &dax_fops);
340 if (cdev_add(&c_dev, first, 1) == -1) {
341 dax_err("cdev_add failed");
342 ret = -ENXIO;
343 goto cdev_error;
344 }
345
346 pr_info("Attached DAX module\n");
347 goto done;
348
349 cdev_error:
350 device_destroy(cl, first);
351 device_error:
352 class_destroy(cl);
353 class_error:
354 unregister_chrdev_region(first, 1);
355 done:
356 mdesc_release(hp);
357 return ret;
358 }
359 module_init(dax_attach);
360
361 static void __exit dax_detach(void)
362 {
363 pr_info("Cleaning up DAX module\n");
364 cdev_del(&c_dev);
365 device_destroy(cl, first);
366 class_destroy(cl);
367 unregister_chrdev_region(first, 1);
368 }
369 module_exit(dax_detach);
370
371 /* map completion area */
372 static int dax_devmap(struct file *f, struct vm_area_struct *vma)
373 {
374 struct dax_ctx *ctx = (struct dax_ctx *)f->private_data;
375 size_t len = vma->vm_end - vma->vm_start;
376
377 dax_dbg("len=0x%lx, flags=0x%lx", len, vma->vm_flags);
378
379 if (ctx->owner != current) {
380 dax_dbg("devmap called from wrong thread");
381 return -EINVAL;
382 }
383
384 if (len != DAX_MMAP_LEN) {
385 dax_dbg("len(%lu) != DAX_MMAP_LEN(%d)", len, DAX_MMAP_LEN);
386 return -EINVAL;
387 }
388
389 /* completion area is mapped read-only for user */
390 if (vma->vm_flags & VM_WRITE)
391 return -EPERM;
392 vma->vm_flags &= ~VM_MAYWRITE;
393
394 if (remap_pfn_range(vma, vma->vm_start, ctx->ca_buf_ra >> PAGE_SHIFT,
395 len, vma->vm_page_prot))
396 return -EAGAIN;
397
398 dax_dbg("mmapped completion area at uva 0x%lx", vma->vm_start);
399 return 0;
400 }
401
402 /* Unlock user pages. Called during dequeue or device close */
403 static void dax_unlock_pages(struct dax_ctx *ctx, int ccb_index, int nelem)
404 {
405 int i, j;
406
407 for (i = ccb_index; i < ccb_index + nelem; i++) {
408 for (j = 0; j < NUM_STREAM_TYPES; j++) {
409 struct page *p = ctx->pages[i][j];
410
411 if (p) {
412 dax_dbg("freeing page %p", p);
413 if (j == OUT)
414 set_page_dirty(p);
415 put_page(p);
416 ctx->pages[i][j] = NULL;
417 }
418 }
419 }
420 }
421
422 static int dax_lock_page(void *va, struct page **p)
423 {
424 int ret;
425
426 dax_dbg("uva %p", va);
427
428 ret = get_user_pages_fast((unsigned long)va, 1, FOLL_WRITE, p);
429 if (ret == 1) {
430 dax_dbg("locked page %p, for VA %p", *p, va);
431 return 0;
432 }
433
434 dax_dbg("get_user_pages failed, va=%p, ret=%d", va, ret);
435 return -1;
436 }
437
438 static int dax_lock_pages(struct dax_ctx *ctx, int idx,
439 int nelem, u64 *err_va)
440 {
441 int i;
442
443 for (i = 0; i < nelem; i++) {
444 struct dax_ccb *ccbp = &ctx->ccb_buf[i];
445
446 /*
447 * For each address in the CCB whose type is virtual,
448 * lock the page and change the type to virtual alternate
449 * context. On error, return the offending address in
450 * err_va.
451 */
452 if (ccbp->hdr.out_addr_type == DAX_ADDR_TYPE_VA) {
453 dax_dbg("output");
454 if (dax_lock_page(ccbp->out,
455 &ctx->pages[i + idx][OUT]) != 0) {
456 *err_va = (u64)ccbp->out;
457 goto error;
458 }
459 ccbp->hdr.out_addr_type = DAX_ADDR_TYPE_VA_ALT;
460 }
461
462 if (ccbp->hdr.pri_addr_type == DAX_ADDR_TYPE_VA) {
463 dax_dbg("input");
464 if (dax_lock_page(ccbp->pri,
465 &ctx->pages[i + idx][PRI]) != 0) {
466 *err_va = (u64)ccbp->pri;
467 goto error;
468 }
469 ccbp->hdr.pri_addr_type = DAX_ADDR_TYPE_VA_ALT;
470 }
471
472 if (ccbp->hdr.sec_addr_type == DAX_ADDR_TYPE_VA) {
473 dax_dbg("sec input");
474 if (dax_lock_page(ccbp->sec,
475 &ctx->pages[i + idx][SEC]) != 0) {
476 *err_va = (u64)ccbp->sec;
477 goto error;
478 }
479 ccbp->hdr.sec_addr_type = DAX_ADDR_TYPE_VA_ALT;
480 }
481
482 if (ccbp->hdr.table_addr_type == DAX_ADDR_TYPE_VA) {
483 dax_dbg("tbl");
484 if (dax_lock_page(ccbp->tbl,
485 &ctx->pages[i + idx][TBL]) != 0) {
486 *err_va = (u64)ccbp->tbl;
487 goto error;
488 }
489 ccbp->hdr.table_addr_type = DAX_ADDR_TYPE_VA_ALT;
490 }
491
492 /* skip over 2nd 64 bytes of long CCB */
493 if (ccbp->hdr.longccb)
494 i++;
495 }
496 return DAX_SUBMIT_OK;
497
498 error:
499 dax_unlock_pages(ctx, idx, nelem);
500 return DAX_SUBMIT_ERR_NOACCESS;
501 }
502
503 static void dax_ccb_wait(struct dax_ctx *ctx, int idx)
504 {
505 int ret, nretries;
506 u16 kill_res;
507
508 dax_dbg("idx=%d", idx);
509
510 for (nretries = 0; nretries < DAX_CCB_RETRIES; nretries++) {
511 if (ctx->ca_buf[idx].status == CCA_STAT_NOT_COMPLETED)
512 udelay(DAX_CCB_USEC);
513 else
514 return;
515 }
516 dax_dbg("ctx (%p): CCB[%d] timed out, wait usec=%d, retries=%d. Killing ccb",
517 (void *)ctx, idx, DAX_CCB_USEC, DAX_CCB_RETRIES);
518
519 ret = dax_ccb_kill(ctx->ca_buf_ra + idx * sizeof(struct dax_cca),
520 &kill_res);
521 dax_dbg("Kill CCB[%d] %s", idx, ret ? "failed" : "succeeded");
522 }
523
524 static int dax_close(struct inode *ino, struct file *f)
525 {
526 struct dax_ctx *ctx = (struct dax_ctx *)f->private_data;
527 int i;
528
529 f->private_data = NULL;
530
531 for (i = 0; i < DAX_CA_ELEMS; i++) {
532 if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) {
533 dax_dbg("CCB[%d] not completed", i);
534 dax_ccb_wait(ctx, i);
535 }
536 dax_unlock_pages(ctx, i, 1);
537 }
538
539 kfree(ctx->ccb_buf);
540 kfree(ctx->ca_buf);
541 dax_stat_dbg("CCBs: %d good, %d bad", ctx->ccb_count, ctx->fail_count);
542 kfree(ctx);
543
544 return 0;
545 }
546
547 static ssize_t dax_read(struct file *f, char __user *buf,
548 size_t count, loff_t *ppos)
549 {
550 struct dax_ctx *ctx = f->private_data;
551
552 if (ctx->client != current)
553 return -EUSERS;
554
555 ctx->client = NULL;
556
557 if (count != sizeof(union ccb_result))
558 return -EINVAL;
559 if (copy_to_user(buf, &ctx->result, sizeof(union ccb_result)))
560 return -EFAULT;
561 return count;
562 }
563
564 static ssize_t dax_write(struct file *f, const char __user *buf,
565 size_t count, loff_t *ppos)
566 {
567 struct dax_ctx *ctx = f->private_data;
568 struct dax_command hdr;
569 unsigned long ca;
570 int i, idx, ret;
571
572 if (ctx->client != NULL)
573 return -EINVAL;
574
575 if (count == 0 || count > DAX_MAX_CCBS * sizeof(struct dax_ccb))
576 return -EINVAL;
577
578 if (count % sizeof(struct dax_ccb) == 0)
579 return dax_ccb_exec(ctx, buf, count, ppos); /* CCB EXEC */
580
581 if (count != sizeof(struct dax_command))
582 return -EINVAL;
583
584 /* immediate command */
585 if (ctx->owner != current)
586 return -EUSERS;
587
588 if (copy_from_user(&hdr, buf, sizeof(hdr)))
589 return -EFAULT;
590
591 ca = ctx->ca_buf_ra + hdr.ca_offset;
592
593 switch (hdr.command) {
594 case CCB_KILL:
595 if (hdr.ca_offset >= DAX_MMAP_LEN) {
596 dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)",
597 hdr.ca_offset, DAX_MMAP_LEN);
598 return -EINVAL;
599 }
600
601 ret = dax_ccb_kill(ca, &ctx->result.kill.action);
602 if (ret != 0) {
603 dax_dbg("dax_ccb_kill failed (ret=%d)", ret);
604 return ret;
605 }
606
607 dax_info_dbg("killed (ca_offset %d)", hdr.ca_offset);
608 idx = hdr.ca_offset / sizeof(struct dax_cca);
609 ctx->ca_buf[idx].status = CCA_STAT_KILLED;
610 ctx->ca_buf[idx].err = CCA_ERR_KILLED;
611 ctx->client = current;
612 return count;
613
614 case CCB_INFO:
615 if (hdr.ca_offset >= DAX_MMAP_LEN) {
616 dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)",
617 hdr.ca_offset, DAX_MMAP_LEN);
618 return -EINVAL;
619 }
620
621 ret = dax_ccb_info(ca, &ctx->result.info);
622 if (ret != 0) {
623 dax_dbg("dax_ccb_info failed (ret=%d)", ret);
624 return ret;
625 }
626
627 dax_info_dbg("info succeeded on ca_offset %d", hdr.ca_offset);
628 ctx->client = current;
629 return count;
630
631 case CCB_DEQUEUE:
632 for (i = 0; i < DAX_CA_ELEMS; i++) {
633 if (ctx->ca_buf[i].status !=
634 CCA_STAT_NOT_COMPLETED)
635 dax_unlock_pages(ctx, i, 1);
636 }
637 return count;
638
639 default:
640 return -EINVAL;
641 }
642 }
643
644 static int dax_open(struct inode *inode, struct file *f)
645 {
646 struct dax_ctx *ctx = NULL;
647 int i;
648
649 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
650 if (ctx == NULL)
651 goto done;
652
653 ctx->ccb_buf = kcalloc(DAX_MAX_CCBS, sizeof(struct dax_ccb),
654 GFP_KERNEL);
655 if (ctx->ccb_buf == NULL)
656 goto done;
657
658 ctx->ccb_buf_ra = virt_to_phys(ctx->ccb_buf);
659 dax_dbg("ctx->ccb_buf=0x%p, ccb_buf_ra=0x%llx",
660 (void *)ctx->ccb_buf, ctx->ccb_buf_ra);
661
662 /* allocate CCB completion area buffer */
663 ctx->ca_buf = kzalloc(DAX_MMAP_LEN, GFP_KERNEL);
664 if (ctx->ca_buf == NULL)
665 goto alloc_error;
666 for (i = 0; i < DAX_CA_ELEMS; i++)
667 ctx->ca_buf[i].status = CCA_STAT_COMPLETED;
668
669 ctx->ca_buf_ra = virt_to_phys(ctx->ca_buf);
670 dax_dbg("ctx=0x%p, ctx->ca_buf=0x%p, ca_buf_ra=0x%llx",
671 (void *)ctx, (void *)ctx->ca_buf, ctx->ca_buf_ra);
672
673 ctx->owner = current;
674 f->private_data = ctx;
675 return 0;
676
677 alloc_error:
678 kfree(ctx->ccb_buf);
679 done:
680 kfree(ctx);
681 return -ENOMEM;
682 }
683
684 static char *dax_hv_errno(unsigned long hv_ret, int *ret)
685 {
686 switch (hv_ret) {
687 case HV_EBADALIGN:
688 *ret = -EFAULT;
689 return "HV_EBADALIGN";
690 case HV_ENORADDR:
691 *ret = -EFAULT;
692 return "HV_ENORADDR";
693 case HV_EINVAL:
694 *ret = -EINVAL;
695 return "HV_EINVAL";
696 case HV_EWOULDBLOCK:
697 *ret = -EAGAIN;
698 return "HV_EWOULDBLOCK";
699 case HV_ENOACCESS:
700 *ret = -EPERM;
701 return "HV_ENOACCESS";
702 default:
703 break;
704 }
705
706 *ret = -EIO;
707 return "UNKNOWN";
708 }
709
710 static int dax_ccb_kill(u64 ca, u16 *kill_res)
711 {
712 unsigned long hv_ret;
713 int count, ret = 0;
714 char *err_str;
715
716 for (count = 0; count < DAX_CCB_RETRIES; count++) {
717 dax_dbg("attempting kill on ca_ra 0x%llx", ca);
718 hv_ret = sun4v_ccb_kill(ca, kill_res);
719
720 if (hv_ret == HV_EOK) {
721 dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca,
722 *kill_res);
723 } else {
724 err_str = dax_hv_errno(hv_ret, &ret);
725 dax_dbg("%s (ca_ra 0x%llx)", err_str, ca);
726 }
727
728 if (ret != -EAGAIN)
729 return ret;
730 dax_info_dbg("ccb_kill count = %d", count);
731 udelay(DAX_CCB_USEC);
732 }
733
734 return -EAGAIN;
735 }
736
737 static int dax_ccb_info(u64 ca, struct ccb_info_result *info)
738 {
739 unsigned long hv_ret;
740 char *err_str;
741 int ret = 0;
742
743 dax_dbg("attempting info on ca_ra 0x%llx", ca);
744 hv_ret = sun4v_ccb_info(ca, info);
745
746 if (hv_ret == HV_EOK) {
747 dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca, info->state);
748 if (info->state == DAX_CCB_ENQUEUED) {
749 dax_info_dbg("dax_unit %d, queue_num %d, queue_pos %d",
750 info->inst_num, info->q_num, info->q_pos);
751 }
752 } else {
753 err_str = dax_hv_errno(hv_ret, &ret);
754 dax_dbg("%s (ca_ra 0x%llx)", err_str, ca);
755 }
756
757 return ret;
758 }
759
760 static void dax_prt_ccbs(struct dax_ccb *ccb, int nelem)
761 {
762 int i, j;
763 u64 *ccbp;
764
765 dax_dbg("ccb buffer:");
766 for (i = 0; i < nelem; i++) {
767 ccbp = (u64 *)&ccb[i];
768 dax_dbg(" %sccb[%d]", ccb[i].hdr.longccb ? "long " : "", i);
769 for (j = 0; j < 8; j++)
770 dax_dbg("\tccb[%d].dwords[%d]=0x%llx",
771 i, j, *(ccbp + j));
772 }
773 }
774
775 /*
776 * Validates user CCB content. Also sets completion address and address types
777 * for all addresses contained in CCB.
778 */
779 static int dax_preprocess_usr_ccbs(struct dax_ctx *ctx, int idx, int nelem)
780 {
781 int i;
782
783 /*
784 * The user is not allowed to specify real address types in
785 * the CCB header. This must be enforced by the kernel before
786 * submitting the CCBs to HV. The only allowed values for all
787 * address fields are VA or IMM
788 */
789 for (i = 0; i < nelem; i++) {
790 struct dax_ccb *ccbp = &ctx->ccb_buf[i];
791 unsigned long ca_offset;
792
793 if (ccbp->hdr.ccb_version > max_ccb_version)
794 return DAX_SUBMIT_ERR_CCB_INVAL;
795
796 switch (ccbp->hdr.opcode) {
797 case DAX_OP_SYNC_NOP:
798 case DAX_OP_EXTRACT:
799 case DAX_OP_SCAN_VALUE:
800 case DAX_OP_SCAN_RANGE:
801 case DAX_OP_TRANSLATE:
802 case DAX_OP_SCAN_VALUE | DAX_OP_INVERT:
803 case DAX_OP_SCAN_RANGE | DAX_OP_INVERT:
804 case DAX_OP_TRANSLATE | DAX_OP_INVERT:
805 case DAX_OP_SELECT:
806 break;
807 default:
808 return DAX_SUBMIT_ERR_CCB_INVAL;
809 }
810
811 if (ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_VA &&
812 ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_NONE) {
813 dax_dbg("invalid out_addr_type in user CCB[%d]", i);
814 return DAX_SUBMIT_ERR_CCB_INVAL;
815 }
816
817 if (ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_VA &&
818 ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_NONE) {
819 dax_dbg("invalid pri_addr_type in user CCB[%d]", i);
820 return DAX_SUBMIT_ERR_CCB_INVAL;
821 }
822
823 if (ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_VA &&
824 ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_NONE) {
825 dax_dbg("invalid sec_addr_type in user CCB[%d]", i);
826 return DAX_SUBMIT_ERR_CCB_INVAL;
827 }
828
829 if (ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_VA &&
830 ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_NONE) {
831 dax_dbg("invalid table_addr_type in user CCB[%d]", i);
832 return DAX_SUBMIT_ERR_CCB_INVAL;
833 }
834
835 /* set completion (real) address and address type */
836 ccbp->hdr.cca_addr_type = DAX_ADDR_TYPE_RA;
837 ca_offset = (idx + i) * sizeof(struct dax_cca);
838 ccbp->ca = (void *)ctx->ca_buf_ra + ca_offset;
839 memset(&ctx->ca_buf[idx + i], 0, sizeof(struct dax_cca));
840
841 dax_dbg("ccb[%d]=%p, ca_offset=0x%lx, compl RA=0x%llx",
842 i, ccbp, ca_offset, ctx->ca_buf_ra + ca_offset);
843
844 /* skip over 2nd 64 bytes of long CCB */
845 if (ccbp->hdr.longccb)
846 i++;
847 }
848
849 return DAX_SUBMIT_OK;
850 }
851
852 static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf,
853 size_t count, loff_t *ppos)
854 {
855 unsigned long accepted_len, hv_rv;
856 int i, idx, nccbs, naccepted;
857
858 ctx->client = current;
859 idx = *ppos;
860 nccbs = count / sizeof(struct dax_ccb);
861
862 if (ctx->owner != current) {
863 dax_dbg("wrong thread");
864 ctx->result.exec.status = DAX_SUBMIT_ERR_THR_INIT;
865 return 0;
866 }
867 dax_dbg("args: ccb_buf_len=%ld, idx=%d", count, idx);
868
869 /* for given index and length, verify ca_buf range exists */
870 if (idx < 0 || idx > (DAX_CA_ELEMS - nccbs)) {
871 ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;
872 return 0;
873 }
874
875 /*
876 * Copy CCBs into kernel buffer to prevent modification by the
877 * user in between validation and submission.
878 */
879 if (copy_from_user(ctx->ccb_buf, buf, count)) {
880 dax_dbg("copyin of user CCB buffer failed");
881 ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS;
882 return 0;
883 }
884
885 /* check to see if ca_buf[idx] .. ca_buf[idx + nccbs] are available */
886 for (i = idx; i < idx + nccbs; i++) {
887 if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) {
888 dax_dbg("CA range not available, dequeue needed");
889 ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL;
890 return 0;
891 }
892 }
893 dax_unlock_pages(ctx, idx, nccbs);
894
895 ctx->result.exec.status = dax_preprocess_usr_ccbs(ctx, idx, nccbs);
896 if (ctx->result.exec.status != DAX_SUBMIT_OK)
897 return 0;
898
899 ctx->result.exec.status = dax_lock_pages(ctx, idx, nccbs,
900 &ctx->result.exec.status_data);
901 if (ctx->result.exec.status != DAX_SUBMIT_OK)
902 return 0;
903
904 if (dax_debug & DAX_DBG_FLG_BASIC)
905 dax_prt_ccbs(ctx->ccb_buf, nccbs);
906
907 hv_rv = sun4v_ccb_submit(ctx->ccb_buf_ra, count,
908 HV_CCB_QUERY_CMD | HV_CCB_VA_SECONDARY, 0,
909 &accepted_len, &ctx->result.exec.status_data);
910
911 switch (hv_rv) {
912 case HV_EOK:
913 /*
914 * Hcall succeeded with no errors but the accepted
915 * length may be less than the requested length. The
916 * only way the driver can resubmit the remainder is
917 * to wait for completion of the submitted CCBs since
918 * there is no way to guarantee the ordering semantics
919 * required by the client applications. Therefore we
920 * let the user library deal with resubmissions.
921 */
922 ctx->result.exec.status = DAX_SUBMIT_OK;
923 break;
924 case HV_EWOULDBLOCK:
925 /*
926 * This is a transient HV API error. The user library
927 * can retry.
928 */
929 dax_dbg("hcall returned HV_EWOULDBLOCK");
930 ctx->result.exec.status = DAX_SUBMIT_ERR_WOULDBLOCK;
931 break;
932 case HV_ENOMAP:
933 /*
934 * HV was unable to translate a VA. The VA it could
935 * not translate is returned in the status_data param.
936 */
937 dax_dbg("hcall returned HV_ENOMAP");
938 ctx->result.exec.status = DAX_SUBMIT_ERR_NOMAP;
939 break;
940 case HV_EINVAL:
941 /*
942 * This is the result of an invalid user CCB as HV is
943 * validating some of the user CCB fields. Pass this
944 * error back to the user. There is no supporting info
945 * to isolate the invalid field.
946 */
947 dax_dbg("hcall returned HV_EINVAL");
948 ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_INVAL;
949 break;
950 case HV_ENOACCESS:
951 /*
952 * HV found a VA that did not have the appropriate
953 * permissions (such as the w bit). The VA in question
954 * is returned in status_data param.
955 */
956 dax_dbg("hcall returned HV_ENOACCESS");
957 ctx->result.exec.status = DAX_SUBMIT_ERR_NOACCESS;
958 break;
959 case HV_EUNAVAILABLE:
960 /*
961 * The requested CCB operation could not be performed
962 * at this time. Return the specific unavailable code
963 * in the status_data field.
964 */
965 dax_dbg("hcall returned HV_EUNAVAILABLE");
966 ctx->result.exec.status = DAX_SUBMIT_ERR_UNAVAIL;
967 break;
968 default:
969 ctx->result.exec.status = DAX_SUBMIT_ERR_INTERNAL;
970 dax_dbg("unknown hcall return value (%ld)", hv_rv);
971 break;
972 }
973
974 /* unlock pages associated with the unaccepted CCBs */
975 naccepted = accepted_len / sizeof(struct dax_ccb);
976 dax_unlock_pages(ctx, idx + naccepted, nccbs - naccepted);
977
978 /* mark unaccepted CCBs as not completed */
979 for (i = idx + naccepted; i < idx + nccbs; i++)
980 ctx->ca_buf[i].status = CCA_STAT_COMPLETED;
981
982 ctx->ccb_count += naccepted;
983 ctx->fail_count += nccbs - naccepted;
984
985 dax_dbg("hcall rv=%ld, accepted_len=%ld, status_data=0x%llx, ret status=%d",
986 hv_rv, accepted_len, ctx->result.exec.status_data,
987 ctx->result.exec.status);
988
989 if (count == accepted_len)
990 ctx->client = NULL; /* no read needed to complete protocol */
991 return accepted_len;
992 }
Linux with added FPC-III support