?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / drivers / hwtracing / ptt / hisi_ptt.c
blob3090479a2979883356edf9027ffb22aeeb371674
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Driver for HiSilicon PCIe tune and trace device
4 *
5 * Copyright (c) 2022 HiSilicon Technologies Co., Ltd.
6 * Author: Yicong Yang <yangyicong@hisilicon.com>
7 */
8
9 #include <linux/bitfield.h>
10 #include <linux/bitops.h>
11 #include <linux/cpuhotplug.h>
12 #include <linux/delay.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/interrupt.h>
15 #include <linux/io.h>
16 #include <linux/iommu.h>
17 #include <linux/iopoll.h>
18 #include <linux/module.h>
19 #include <linux/sysfs.h>
20 #include <linux/vmalloc.h>
21
22 #include "hisi_ptt.h"
23
24 /* Dynamic CPU hotplug state used by PTT */
25 static enum cpuhp_state hisi_ptt_pmu_online;
26
27 static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt)
28 {
29 u32 val;
30
31 return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT,
32 val, !(val & HISI_PTT_TUNING_INT_STAT_MASK),
33 HISI_PTT_WAIT_POLL_INTERVAL_US,
34 HISI_PTT_WAIT_TUNE_TIMEOUT_US);
35 }
36
37 static ssize_t hisi_ptt_tune_attr_show(struct device *dev,
38 struct device_attribute *attr,
39 char *buf)
40 {
41 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
42 struct dev_ext_attribute *ext_attr;
43 struct hisi_ptt_tune_desc *desc;
44 u32 reg;
45 u16 val;
46
47 ext_attr = container_of(attr, struct dev_ext_attribute, attr);
48 desc = ext_attr->var;
49
50 mutex_lock(&hisi_ptt->tune_lock);
51
52 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
53 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);
54 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,
55 desc->event_code);
56 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
57
58 /* Write all 1 to indicates it's the read process */
59 writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
60
61 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {
62 mutex_unlock(&hisi_ptt->tune_lock);
63 return -ETIMEDOUT;
64 }
65
66 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
67 reg &= HISI_PTT_TUNING_DATA_VAL_MASK;
68 val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg);
69
70 mutex_unlock(&hisi_ptt->tune_lock);
71 return sysfs_emit(buf, "%u\n", val);
72 }
73
74 static ssize_t hisi_ptt_tune_attr_store(struct device *dev,
75 struct device_attribute *attr,
76 const char *buf, size_t count)
77 {
78 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
79 struct dev_ext_attribute *ext_attr;
80 struct hisi_ptt_tune_desc *desc;
81 u32 reg;
82 u16 val;
83
84 ext_attr = container_of(attr, struct dev_ext_attribute, attr);
85 desc = ext_attr->var;
86
87 if (kstrtou16(buf, 10, &val))
88 return -EINVAL;
89
90 mutex_lock(&hisi_ptt->tune_lock);
91
92 reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
93 reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB);
94 reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB,
95 desc->event_code);
96 writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL);
97 writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val),
98 hisi_ptt->iobase + HISI_PTT_TUNING_DATA);
99
100 if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) {
101 mutex_unlock(&hisi_ptt->tune_lock);
102 return -ETIMEDOUT;
103 }
104
105 mutex_unlock(&hisi_ptt->tune_lock);
106 return count;
107 }
108
109 #define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \
110 static struct hisi_ptt_tune_desc _name##_desc = { \
111 .name = #_name, \
112 .event_code = (_val), \
113 }; \
114 static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \
115 .attr = __ATTR(_name, 0600, _show, _store), \
116 .var = &_name##_desc, \
117 }
118
119 #define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \
120 HISI_PTT_TUNE_ATTR(_name, _val, \
121 hisi_ptt_tune_attr_show, \
122 hisi_ptt_tune_attr_store)
123
124 /*
125 * The value of the tuning event are composed of two parts: main event code
126 * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is
127 * a subevent of 'Tx path QoS control' which for tuning the weight of Tx
128 * completion TLPs. See hisi_ptt.rst documentation for more information.
129 */
130 #define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16))
131 #define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16))
132 #define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16))
133 #define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16))
134 #define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16))
135
136 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL);
137 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP);
138 HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P);
139 HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL);
140 HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL);
141
142 static struct attribute *hisi_ptt_tune_attrs[] = {
143 &hisi_ptt_qos_tx_cpl_attr.attr.attr,
144 &hisi_ptt_qos_tx_np_attr.attr.attr,
145 &hisi_ptt_qos_tx_p_attr.attr.attr,
146 &hisi_ptt_rx_alloc_buf_level_attr.attr.attr,
147 &hisi_ptt_tx_alloc_buf_level_attr.attr.attr,
148 NULL,
149 };
150
151 static struct attribute_group hisi_ptt_tune_group = {
152 .name = "tune",
153 .attrs = hisi_ptt_tune_attrs,
154 };
155
156 static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port)
157 {
158 if (is_port)
159 return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff));
160
161 return devid;
162 }
163
164 static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt)
165 {
166 u32 val;
167
168 return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS,
169 val, val & HISI_PTT_TRACE_IDLE,
170 HISI_PTT_WAIT_POLL_INTERVAL_US,
171 HISI_PTT_WAIT_TRACE_TIMEOUT_US);
172 }
173
174 static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt)
175 {
176 u32 val;
177
178 readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS,
179 val, !val, HISI_PTT_RESET_POLL_INTERVAL_US,
180 HISI_PTT_RESET_TIMEOUT_US);
181 }
182
183 static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt)
184 {
185 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
186
187 /* Mask the interrupt on the end */
188 writel(HISI_PTT_TRACE_INT_MASK_ALL, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK);
189
190 hisi_ptt->trace_ctrl.started = false;
191 }
192
193 static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt)
194 {
195 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
196 u32 val;
197 int i;
198
199 /* Check device idle before start trace */
200 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) {
201 pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n");
202 return -EBUSY;
203 }
204
205 ctrl->started = true;
206
207 /* Reset the DMA before start tracing */
208 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
209 val |= HISI_PTT_TRACE_CTRL_RST;
210 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
211
212 hisi_ptt_wait_dma_reset_done(hisi_ptt);
213
214 val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
215 val &= ~HISI_PTT_TRACE_CTRL_RST;
216 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
217
218 /* Reset the index of current buffer */
219 hisi_ptt->trace_ctrl.buf_index = 0;
220
221 /* Zero the trace buffers */
222 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++)
223 memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE);
224
225 /* Clear the interrupt status */
226 writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
227 writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK);
228
229 /* Set the trace control register */
230 val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type);
231 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction);
232 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format);
233 val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter);
234 if (!hisi_ptt->trace_ctrl.is_port)
235 val |= HISI_PTT_TRACE_CTRL_FILTER_MODE;
236
237 /* Start the Trace */
238 val |= HISI_PTT_TRACE_CTRL_EN;
239 writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
240
241 return 0;
242 }
243
244 static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop)
245 {
246 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
247 struct perf_output_handle *handle = &ctrl->handle;
248 struct perf_event *event = handle->event;
249 struct hisi_ptt_pmu_buf *buf;
250 size_t size;
251 void *addr;
252
253 buf = perf_get_aux(handle);
254 if (!buf || !handle->size)
255 return -EINVAL;
256
257 addr = ctrl->trace_buf[ctrl->buf_index].addr;
258
259 /*
260 * If we're going to stop, read the size of already traced data from
261 * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt,
262 * the data size is always HISI_PTT_TRACE_BUF_SIZE.
263 */
264 if (stop) {
265 u32 reg;
266
267 reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS);
268 size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg);
269 } else {
270 size = HISI_PTT_TRACE_BUF_SIZE;
271 }
272
273 memcpy(buf->base + buf->pos, addr, size);
274 buf->pos += size;
275
276 /*
277 * Always commit the data to the AUX buffer in time to make sure
278 * userspace got enough time to consume the data.
279 *
280 * If we're not going to stop, apply a new one and check whether
281 * there's enough room for the next trace.
282 */
283 perf_aux_output_end(handle, size);
284 if (!stop) {
285 buf = perf_aux_output_begin(handle, event);
286 if (!buf)
287 return -EINVAL;
288
289 buf->pos = handle->head % buf->length;
290 if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {
291 perf_aux_output_end(handle, 0);
292 return -EINVAL;
293 }
294 }
295
296 return 0;
297 }
298
299 static irqreturn_t hisi_ptt_isr(int irq, void *context)
300 {
301 struct hisi_ptt *hisi_ptt = context;
302 u32 status, buf_idx;
303
304 status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
305 if (!(status & HISI_PTT_TRACE_INT_STAT_MASK))
306 return IRQ_NONE;
307
308 buf_idx = ffs(status) - 1;
309
310 /* Clear the interrupt status of buffer @buf_idx */
311 writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT);
312
313 /*
314 * Update the AUX buffer and cache the current buffer index,
315 * as we need to know this and save the data when the trace
316 * is ended out of the interrupt handler. End the trace
317 * if the updating fails.
318 */
319 if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false))
320 hisi_ptt_trace_end(hisi_ptt);
321 else
322 hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT;
323
324 return IRQ_HANDLED;
325 }
326
327 static void hisi_ptt_irq_free_vectors(void *pdev)
328 {
329 pci_free_irq_vectors(pdev);
330 }
331
332 static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt)
333 {
334 struct pci_dev *pdev = hisi_ptt->pdev;
335 int ret;
336
337 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
338 if (ret < 0) {
339 pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret);
340 return ret;
341 }
342
343 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev);
344 if (ret < 0)
345 return ret;
346
347 hisi_ptt->trace_irq = pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ);
348 ret = devm_request_irq(&pdev->dev, hisi_ptt->trace_irq, hisi_ptt_isr,
349 IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME,
350 hisi_ptt);
351 if (ret) {
352 pci_err(pdev, "failed to request irq %d, ret = %d\n",
353 hisi_ptt->trace_irq, ret);
354 return ret;
355 }
356
357 return 0;
358 }
359
360 static void hisi_ptt_del_free_filter(struct hisi_ptt *hisi_ptt,
361 struct hisi_ptt_filter_desc *filter)
362 {
363 if (filter->is_port)
364 hisi_ptt->port_mask &= ~hisi_ptt_get_filter_val(filter->devid, true);
365
366 list_del(&filter->list);
367 kfree(filter->name);
368 kfree(filter);
369 }
370
371 static struct hisi_ptt_filter_desc *
372 hisi_ptt_alloc_add_filter(struct hisi_ptt *hisi_ptt, u16 devid, bool is_port)
373 {
374 struct hisi_ptt_filter_desc *filter;
375 u8 devfn = devid & 0xff;
376 char *filter_name;
377
378 filter_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", pci_domain_nr(hisi_ptt->pdev->bus),
379 PCI_BUS_NUM(devid), PCI_SLOT(devfn), PCI_FUNC(devfn));
380 if (!filter_name) {
381 pci_err(hisi_ptt->pdev, "failed to allocate name for filter %04x:%02x:%02x.%d\n",
382 pci_domain_nr(hisi_ptt->pdev->bus), PCI_BUS_NUM(devid),
383 PCI_SLOT(devfn), PCI_FUNC(devfn));
384 return NULL;
385 }
386
387 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
388 if (!filter) {
389 pci_err(hisi_ptt->pdev, "failed to add filter for %s\n",
390 filter_name);
391 kfree(filter_name);
392 return NULL;
393 }
394
395 filter->name = filter_name;
396 filter->is_port = is_port;
397 filter->devid = devid;
398
399 if (filter->is_port) {
400 list_add_tail(&filter->list, &hisi_ptt->port_filters);
401
402 /* Update the available port mask */
403 hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true);
404 } else {
405 list_add_tail(&filter->list, &hisi_ptt->req_filters);
406 }
407
408 return filter;
409 }
410
411 static ssize_t hisi_ptt_filter_show(struct device *dev, struct device_attribute *attr,
412 char *buf)
413 {
414 struct hisi_ptt_filter_desc *filter;
415 unsigned long filter_val;
416
417 filter = container_of(attr, struct hisi_ptt_filter_desc, attr);
418 filter_val = hisi_ptt_get_filter_val(filter->devid, filter->is_port) |
419 (filter->is_port ? HISI_PTT_PMU_FILTER_IS_PORT : 0);
420
421 return sysfs_emit(buf, "0x%05lx\n", filter_val);
422 }
423
424 static int hisi_ptt_create_rp_filter_attr(struct hisi_ptt *hisi_ptt,
425 struct hisi_ptt_filter_desc *filter)
426 {
427 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
428
429 sysfs_attr_init(&filter->attr.attr);
430 filter->attr.attr.name = filter->name;
431 filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */
432 filter->attr.show = hisi_ptt_filter_show;
433
434 return sysfs_add_file_to_group(kobj, &filter->attr.attr,
435 HISI_PTT_RP_FILTERS_GRP_NAME);
436 }
437
438 static void hisi_ptt_remove_rp_filter_attr(struct hisi_ptt *hisi_ptt,
439 struct hisi_ptt_filter_desc *filter)
440 {
441 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
442
443 sysfs_remove_file_from_group(kobj, &filter->attr.attr,
444 HISI_PTT_RP_FILTERS_GRP_NAME);
445 }
446
447 static int hisi_ptt_create_req_filter_attr(struct hisi_ptt *hisi_ptt,
448 struct hisi_ptt_filter_desc *filter)
449 {
450 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
451
452 sysfs_attr_init(&filter->attr.attr);
453 filter->attr.attr.name = filter->name;
454 filter->attr.attr.mode = 0400; /* DEVICE_ATTR_ADMIN_RO */
455 filter->attr.show = hisi_ptt_filter_show;
456
457 return sysfs_add_file_to_group(kobj, &filter->attr.attr,
458 HISI_PTT_REQ_FILTERS_GRP_NAME);
459 }
460
461 static void hisi_ptt_remove_req_filter_attr(struct hisi_ptt *hisi_ptt,
462 struct hisi_ptt_filter_desc *filter)
463 {
464 struct kobject *kobj = &hisi_ptt->hisi_ptt_pmu.dev->kobj;
465
466 sysfs_remove_file_from_group(kobj, &filter->attr.attr,
467 HISI_PTT_REQ_FILTERS_GRP_NAME);
468 }
469
470 static int hisi_ptt_create_filter_attr(struct hisi_ptt *hisi_ptt,
471 struct hisi_ptt_filter_desc *filter)
472 {
473 int ret;
474
475 if (filter->is_port)
476 ret = hisi_ptt_create_rp_filter_attr(hisi_ptt, filter);
477 else
478 ret = hisi_ptt_create_req_filter_attr(hisi_ptt, filter);
479
480 if (ret)
481 pci_err(hisi_ptt->pdev, "failed to create sysfs attribute for filter %s\n",
482 filter->name);
483
484 return ret;
485 }
486
487 static void hisi_ptt_remove_filter_attr(struct hisi_ptt *hisi_ptt,
488 struct hisi_ptt_filter_desc *filter)
489 {
490 if (filter->is_port)
491 hisi_ptt_remove_rp_filter_attr(hisi_ptt, filter);
492 else
493 hisi_ptt_remove_req_filter_attr(hisi_ptt, filter);
494 }
495
496 static void hisi_ptt_remove_all_filter_attributes(void *data)
497 {
498 struct hisi_ptt_filter_desc *filter;
499 struct hisi_ptt *hisi_ptt = data;
500
501 mutex_lock(&hisi_ptt->filter_lock);
502
503 list_for_each_entry(filter, &hisi_ptt->req_filters, list)
504 hisi_ptt_remove_filter_attr(hisi_ptt, filter);
505
506 list_for_each_entry(filter, &hisi_ptt->port_filters, list)
507 hisi_ptt_remove_filter_attr(hisi_ptt, filter);
508
509 hisi_ptt->sysfs_inited = false;
510 mutex_unlock(&hisi_ptt->filter_lock);
511 }
512
513 static int hisi_ptt_init_filter_attributes(struct hisi_ptt *hisi_ptt)
514 {
515 struct hisi_ptt_filter_desc *filter;
516 int ret;
517
518 mutex_lock(&hisi_ptt->filter_lock);
519
520 /*
521 * Register the reset callback in the first stage. In reset we traverse
522 * the filters list to remove the sysfs attributes so the callback can
523 * be called safely even without below filter attributes creation.
524 */
525 ret = devm_add_action(&hisi_ptt->pdev->dev,
526 hisi_ptt_remove_all_filter_attributes,
527 hisi_ptt);
528 if (ret)
529 goto out;
530
531 list_for_each_entry(filter, &hisi_ptt->port_filters, list) {
532 ret = hisi_ptt_create_filter_attr(hisi_ptt, filter);
533 if (ret)
534 goto out;
535 }
536
537 list_for_each_entry(filter, &hisi_ptt->req_filters, list) {
538 ret = hisi_ptt_create_filter_attr(hisi_ptt, filter);
539 if (ret)
540 goto out;
541 }
542
543 hisi_ptt->sysfs_inited = true;
544 out:
545 mutex_unlock(&hisi_ptt->filter_lock);
546 return ret;
547 }
548
549 static void hisi_ptt_update_filters(struct work_struct *work)
550 {
551 struct delayed_work *delayed_work = to_delayed_work(work);
552 struct hisi_ptt_filter_update_info info;
553 struct hisi_ptt_filter_desc *filter;
554 struct hisi_ptt *hisi_ptt;
555
556 hisi_ptt = container_of(delayed_work, struct hisi_ptt, work);
557
558 if (!mutex_trylock(&hisi_ptt->filter_lock)) {
559 schedule_delayed_work(&hisi_ptt->work, HISI_PTT_WORK_DELAY_MS);
560 return;
561 }
562
563 while (kfifo_get(&hisi_ptt->filter_update_kfifo, &info)) {
564 if (info.is_add) {
565 /*
566 * Notify the users if failed to add this filter, others
567 * still work and available. See the comments in
568 * hisi_ptt_init_filters().
569 */
570 filter = hisi_ptt_alloc_add_filter(hisi_ptt, info.devid, info.is_port);
571 if (!filter)
572 continue;
573
574 /*
575 * If filters' sysfs entries hasn't been initialized,
576 * then we're still at probe stage. Add the filters to
577 * the list and later hisi_ptt_init_filter_attributes()
578 * will create sysfs attributes for all the filters.
579 */
580 if (hisi_ptt->sysfs_inited &&
581 hisi_ptt_create_filter_attr(hisi_ptt, filter)) {
582 hisi_ptt_del_free_filter(hisi_ptt, filter);
583 continue;
584 }
585 } else {
586 struct hisi_ptt_filter_desc *tmp;
587 struct list_head *target_list;
588
589 target_list = info.is_port ? &hisi_ptt->port_filters :
590 &hisi_ptt->req_filters;
591
592 list_for_each_entry_safe(filter, tmp, target_list, list)
593 if (filter->devid == info.devid) {
594 if (hisi_ptt->sysfs_inited)
595 hisi_ptt_remove_filter_attr(hisi_ptt, filter);
596
597 hisi_ptt_del_free_filter(hisi_ptt, filter);
598 break;
599 }
600 }
601 }
602
603 mutex_unlock(&hisi_ptt->filter_lock);
604 }
605
606 /*
607 * A PCI bus notifier is used here for dynamically updating the filter
608 * list.
609 */
610 static int hisi_ptt_notifier_call(struct notifier_block *nb, unsigned long action,
611 void *data)
612 {
613 struct hisi_ptt *hisi_ptt = container_of(nb, struct hisi_ptt, hisi_ptt_nb);
614 struct hisi_ptt_filter_update_info info;
615 struct pci_dev *pdev, *root_port;
616 struct device *dev = data;
617 u32 port_devid;
618
619 pdev = to_pci_dev(dev);
620 root_port = pcie_find_root_port(pdev);
621 if (!root_port)
622 return 0;
623
624 port_devid = pci_dev_id(root_port);
625 if (port_devid < hisi_ptt->lower_bdf ||
626 port_devid > hisi_ptt->upper_bdf)
627 return 0;
628
629 info.is_port = pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT;
630 info.devid = pci_dev_id(pdev);
631
632 switch (action) {
633 case BUS_NOTIFY_ADD_DEVICE:
634 info.is_add = true;
635 break;
636 case BUS_NOTIFY_DEL_DEVICE:
637 info.is_add = false;
638 break;
639 default:
640 return 0;
641 }
642
643 /*
644 * The FIFO size is 16 which is sufficient for almost all the cases,
645 * since each PCIe core will have most 8 Root Ports (typically only
646 * 1~4 Root Ports). On failure log the failed filter and let user
647 * handle it.
648 */
649 if (kfifo_in_spinlocked(&hisi_ptt->filter_update_kfifo, &info, 1,
650 &hisi_ptt->filter_update_lock))
651 schedule_delayed_work(&hisi_ptt->work, 0);
652 else
653 pci_warn(hisi_ptt->pdev,
654 "filter update fifo overflow for target %s\n",
655 pci_name(pdev));
656
657 return 0;
658 }
659
660 static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data)
661 {
662 struct pci_dev *root_port = pcie_find_root_port(pdev);
663 struct hisi_ptt_filter_desc *filter;
664 struct hisi_ptt *hisi_ptt = data;
665 u32 port_devid;
666
667 if (!root_port)
668 return 0;
669
670 port_devid = pci_dev_id(root_port);
671 if (port_devid < hisi_ptt->lower_bdf ||
672 port_devid > hisi_ptt->upper_bdf)
673 return 0;
674
675 /*
676 * We won't fail the probe if filter allocation failed here. The filters
677 * should be partial initialized and users would know which filter fails
678 * through the log. Other functions of PTT device are still available.
679 */
680 filter = hisi_ptt_alloc_add_filter(hisi_ptt, pci_dev_id(pdev),
681 pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT);
682 if (!filter)
683 return -ENOMEM;
684
685 return 0;
686 }
687
688 static void hisi_ptt_release_filters(void *data)
689 {
690 struct hisi_ptt_filter_desc *filter, *tmp;
691 struct hisi_ptt *hisi_ptt = data;
692
693 list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list)
694 hisi_ptt_del_free_filter(hisi_ptt, filter);
695
696 list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list)
697 hisi_ptt_del_free_filter(hisi_ptt, filter);
698 }
699
700 static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt)
701 {
702 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
703 struct device *dev = &hisi_ptt->pdev->dev;
704 int i;
705
706 ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT,
707 sizeof(*ctrl->trace_buf), GFP_KERNEL);
708 if (!ctrl->trace_buf)
709 return -ENOMEM;
710
711 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) {
712 ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE,
713 &ctrl->trace_buf[i].dma,
714 GFP_KERNEL);
715 if (!ctrl->trace_buf[i].addr)
716 return -ENOMEM;
717 }
718
719 /* Configure the trace DMA buffer */
720 for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) {
721 writel(lower_32_bits(ctrl->trace_buf[i].dma),
722 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 +
723 i * HISI_PTT_TRACE_ADDR_STRIDE);
724 writel(upper_32_bits(ctrl->trace_buf[i].dma),
725 hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 +
726 i * HISI_PTT_TRACE_ADDR_STRIDE);
727 }
728 writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE);
729
730 return 0;
731 }
732
733 static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt)
734 {
735 struct pci_dev *pdev = hisi_ptt->pdev;
736 struct pci_bus *bus;
737 int ret;
738 u32 reg;
739
740 INIT_DELAYED_WORK(&hisi_ptt->work, hisi_ptt_update_filters);
741 INIT_KFIFO(hisi_ptt->filter_update_kfifo);
742 spin_lock_init(&hisi_ptt->filter_update_lock);
743
744 INIT_LIST_HEAD(&hisi_ptt->port_filters);
745 INIT_LIST_HEAD(&hisi_ptt->req_filters);
746 mutex_init(&hisi_ptt->filter_lock);
747
748 ret = hisi_ptt_config_trace_buf(hisi_ptt);
749 if (ret)
750 return ret;
751
752 /*
753 * The device range register provides the information about the root
754 * ports which the RCiEP can control and trace. The RCiEP and the root
755 * ports which it supports are on the same PCIe core, with same domain
756 * number but maybe different bus number. The device range register
757 * will tell us which root ports we can support, Bit[31:16] indicates
758 * the upper BDF numbers of the root port, while Bit[15:0] indicates
759 * the lower.
760 */
761 reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE);
762 hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg);
763 hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg);
764
765 bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf));
766 if (bus)
767 pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt);
768
769 ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt);
770 if (ret)
771 return ret;
772
773 hisi_ptt->trace_ctrl.on_cpu = -1;
774 return 0;
775 }
776
777 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
778 char *buf)
779 {
780 struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev));
781 const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev));
782
783 return cpumap_print_to_pagebuf(true, buf, cpumask);
784 }
785 static DEVICE_ATTR_RO(cpumask);
786
787 static struct attribute *hisi_ptt_cpumask_attrs[] = {
788 &dev_attr_cpumask.attr,
789 NULL
790 };
791
792 static const struct attribute_group hisi_ptt_cpumask_attr_group = {
793 .attrs = hisi_ptt_cpumask_attrs,
794 };
795
796 /*
797 * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester
798 * filter. Bit[15:0] indicates the filter value, for Root Port filter it's
799 * a bit mask of desired ports and for Requester filter it's the Requester ID
800 * of the desired PCIe function. Bit[18:16] is reserved for extension.
801 *
802 * See hisi_ptt.rst documentation for detailed information.
803 */
804 PMU_FORMAT_ATTR(filter, "config:0-19");
805 PMU_FORMAT_ATTR(direction, "config:20-23");
806 PMU_FORMAT_ATTR(type, "config:24-31");
807 PMU_FORMAT_ATTR(format, "config:32-35");
808
809 static struct attribute *hisi_ptt_pmu_format_attrs[] = {
810 &format_attr_filter.attr,
811 &format_attr_direction.attr,
812 &format_attr_type.attr,
813 &format_attr_format.attr,
814 NULL
815 };
816
817 static struct attribute_group hisi_ptt_pmu_format_group = {
818 .name = "format",
819 .attrs = hisi_ptt_pmu_format_attrs,
820 };
821
822 static ssize_t hisi_ptt_filter_multiselect_show(struct device *dev,
823 struct device_attribute *attr,
824 char *buf)
825 {
826 struct dev_ext_attribute *ext_attr;
827
828 ext_attr = container_of(attr, struct dev_ext_attribute, attr);
829 return sysfs_emit(buf, "%s\n", (char *)ext_attr->var);
830 }
831
832 static struct dev_ext_attribute root_port_filters_multiselect = {
833 .attr = {
834 .attr = { .name = "multiselect", .mode = 0400 },
835 .show = hisi_ptt_filter_multiselect_show,
836 },
837 .var = "1",
838 };
839
840 static struct attribute *hisi_ptt_pmu_root_ports_attrs[] = {
841 &root_port_filters_multiselect.attr.attr,
842 NULL
843 };
844
845 static struct attribute_group hisi_ptt_pmu_root_ports_group = {
846 .name = HISI_PTT_RP_FILTERS_GRP_NAME,
847 .attrs = hisi_ptt_pmu_root_ports_attrs,
848 };
849
850 static struct dev_ext_attribute requester_filters_multiselect = {
851 .attr = {
852 .attr = { .name = "multiselect", .mode = 0400 },
853 .show = hisi_ptt_filter_multiselect_show,
854 },
855 .var = "0",
856 };
857
858 static struct attribute *hisi_ptt_pmu_requesters_attrs[] = {
859 &requester_filters_multiselect.attr.attr,
860 NULL
861 };
862
863 static struct attribute_group hisi_ptt_pmu_requesters_group = {
864 .name = HISI_PTT_REQ_FILTERS_GRP_NAME,
865 .attrs = hisi_ptt_pmu_requesters_attrs,
866 };
867
868 static const struct attribute_group *hisi_ptt_pmu_groups[] = {
869 &hisi_ptt_cpumask_attr_group,
870 &hisi_ptt_pmu_format_group,
871 &hisi_ptt_tune_group,
872 &hisi_ptt_pmu_root_ports_group,
873 &hisi_ptt_pmu_requesters_group,
874 NULL
875 };
876
877 static int hisi_ptt_trace_valid_direction(u32 val)
878 {
879 /*
880 * The direction values have different effects according to the data
881 * format (specified in the parentheses). TLP set A/B means different
882 * set of TLP types. See hisi_ptt.rst documentation for more details.
883 */
884 static const u32 hisi_ptt_trace_available_direction[] = {
885 0, /* inbound(4DW) or reserved(8DW) */
886 1, /* outbound(4DW) */
887 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */
888 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */
889 };
890 int i;
891
892 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) {
893 if (val == hisi_ptt_trace_available_direction[i])
894 return 0;
895 }
896
897 return -EINVAL;
898 }
899
900 static int hisi_ptt_trace_valid_type(u32 val)
901 {
902 /* Different types can be set simultaneously */
903 static const u32 hisi_ptt_trace_available_type[] = {
904 1, /* posted_request */
905 2, /* non-posted_request */
906 4, /* completion */
907 };
908 int i;
909
910 if (!val)
911 return -EINVAL;
912
913 /*
914 * Walk the available list and clear the valid bits of
915 * the config. If there is any resident bit after the
916 * walk then the config is invalid.
917 */
918 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++)
919 val &= ~hisi_ptt_trace_available_type[i];
920
921 if (val)
922 return -EINVAL;
923
924 return 0;
925 }
926
927 static int hisi_ptt_trace_valid_format(u32 val)
928 {
929 static const u32 hisi_ptt_trace_availble_format[] = {
930 0, /* 4DW */
931 1, /* 8DW */
932 };
933 int i;
934
935 for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) {
936 if (val == hisi_ptt_trace_availble_format[i])
937 return 0;
938 }
939
940 return -EINVAL;
941 }
942
943 static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config)
944 {
945 unsigned long val, port_mask = hisi_ptt->port_mask;
946 struct hisi_ptt_filter_desc *filter;
947 int ret = 0;
948
949 hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config);
950 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config);
951
952 /*
953 * Port filters are defined as bit mask. For port filters, check
954 * the bits in the @val are within the range of hisi_ptt->port_mask
955 * and whether it's empty or not, otherwise user has specified
956 * some unsupported root ports.
957 *
958 * For Requester ID filters, walk the available filter list to see
959 * whether we have one matched.
960 */
961 mutex_lock(&hisi_ptt->filter_lock);
962 if (!hisi_ptt->trace_ctrl.is_port) {
963 list_for_each_entry(filter, &hisi_ptt->req_filters, list) {
964 if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port))
965 goto out;
966 }
967 } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) {
968 goto out;
969 }
970
971 ret = -EINVAL;
972 out:
973 mutex_unlock(&hisi_ptt->filter_lock);
974 return ret;
975 }
976
977 static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event)
978 {
979 struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl;
980 u32 val;
981
982 val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config);
983 hisi_ptt->trace_ctrl.filter = val;
984
985 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);
986 ctrl->direction = val;
987
988 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);
989 ctrl->type = val;
990
991 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);
992 ctrl->format = val;
993 }
994
995 static int hisi_ptt_pmu_event_init(struct perf_event *event)
996 {
997 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
998 int ret;
999 u32 val;
1000
1001 if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type)
1002 return -ENOENT;
1003
1004 if (event->cpu < 0) {
1005 dev_dbg(event->pmu->dev, "Per-task mode not supported\n");
1006 return -EOPNOTSUPP;
1007 }
1008
1009 if (event->attach_state & PERF_ATTACH_TASK)
1010 return -EOPNOTSUPP;
1011
1012 ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config);
1013 if (ret < 0)
1014 return ret;
1015
1016 val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config);
1017 ret = hisi_ptt_trace_valid_direction(val);
1018 if (ret < 0)
1019 return ret;
1020
1021 val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config);
1022 ret = hisi_ptt_trace_valid_type(val);
1023 if (ret < 0)
1024 return ret;
1025
1026 val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config);
1027 return hisi_ptt_trace_valid_format(val);
1028 }
1029
1030 static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages,
1031 int nr_pages, bool overwrite)
1032 {
1033 struct hisi_ptt_pmu_buf *buf;
1034 struct page **pagelist;
1035 int i;
1036
1037 if (overwrite) {
1038 dev_warn(event->pmu->dev, "Overwrite mode is not supported\n");
1039 return NULL;
1040 }
1041
1042 /* If the pages size less than buffers, we cannot start trace */
1043 if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE)
1044 return NULL;
1045
1046 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
1047 if (!buf)
1048 return NULL;
1049
1050 pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL);
1051 if (!pagelist)
1052 goto err;
1053
1054 for (i = 0; i < nr_pages; i++)
1055 pagelist[i] = virt_to_page(pages[i]);
1056
1057 buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL);
1058 if (!buf->base) {
1059 kfree(pagelist);
1060 goto err;
1061 }
1062
1063 buf->nr_pages = nr_pages;
1064 buf->length = nr_pages * PAGE_SIZE;
1065 buf->pos = 0;
1066
1067 kfree(pagelist);
1068 return buf;
1069 err:
1070 kfree(buf);
1071 return NULL;
1072 }
1073
1074 static void hisi_ptt_pmu_free_aux(void *aux)
1075 {
1076 struct hisi_ptt_pmu_buf *buf = aux;
1077
1078 vunmap(buf->base);
1079 kfree(buf);
1080 }
1081
1082 static void hisi_ptt_pmu_start(struct perf_event *event, int flags)
1083 {
1084 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
1085 struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle;
1086 struct hw_perf_event *hwc = &event->hw;
1087 struct device *dev = event->pmu->dev;
1088 struct hisi_ptt_pmu_buf *buf;
1089 int cpu = event->cpu;
1090 int ret;
1091
1092 hwc->state = 0;
1093
1094 /* Serialize the perf process if user specified several CPUs */
1095 spin_lock(&hisi_ptt->pmu_lock);
1096 if (hisi_ptt->trace_ctrl.started) {
1097 dev_dbg(dev, "trace has already started\n");
1098 goto stop;
1099 }
1100
1101 /*
1102 * Handle the interrupt on the same cpu which starts the trace to avoid
1103 * context mismatch. Otherwise we'll trigger the WARN from the perf
1104 * core in event_function_local(). If CPU passed is offline we'll fail
1105 * here, just log it since we can do nothing here.
1106 */
1107 ret = irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(cpu));
1108 if (ret)
1109 dev_warn(dev, "failed to set the affinity of trace interrupt\n");
1110
1111 hisi_ptt->trace_ctrl.on_cpu = cpu;
1112
1113 buf = perf_aux_output_begin(handle, event);
1114 if (!buf) {
1115 dev_dbg(dev, "aux output begin failed\n");
1116 goto stop;
1117 }
1118
1119 buf->pos = handle->head % buf->length;
1120
1121 hisi_ptt_pmu_init_configs(hisi_ptt, event);
1122
1123 ret = hisi_ptt_trace_start(hisi_ptt);
1124 if (ret) {
1125 dev_dbg(dev, "trace start failed, ret = %d\n", ret);
1126 perf_aux_output_end(handle, 0);
1127 goto stop;
1128 }
1129
1130 spin_unlock(&hisi_ptt->pmu_lock);
1131 return;
1132 stop:
1133 event->hw.state |= PERF_HES_STOPPED;
1134 spin_unlock(&hisi_ptt->pmu_lock);
1135 }
1136
1137 static void hisi_ptt_pmu_stop(struct perf_event *event, int flags)
1138 {
1139 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
1140 struct hw_perf_event *hwc = &event->hw;
1141
1142 if (hwc->state & PERF_HES_STOPPED)
1143 return;
1144
1145 spin_lock(&hisi_ptt->pmu_lock);
1146 if (hisi_ptt->trace_ctrl.started) {
1147 hisi_ptt_trace_end(hisi_ptt);
1148
1149 if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt))
1150 dev_warn(event->pmu->dev, "Device is still busy\n");
1151
1152 hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true);
1153 }
1154 spin_unlock(&hisi_ptt->pmu_lock);
1155
1156 hwc->state |= PERF_HES_STOPPED;
1157 perf_event_update_userpage(event);
1158 hwc->state |= PERF_HES_UPTODATE;
1159 }
1160
1161 static int hisi_ptt_pmu_add(struct perf_event *event, int flags)
1162 {
1163 struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu);
1164 struct hw_perf_event *hwc = &event->hw;
1165 int cpu = event->cpu;
1166
1167 /* Only allow the cpus on the device's node to add the event */
1168 if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev))))
1169 return 0;
1170
1171 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1172
1173 if (flags & PERF_EF_START) {
1174 hisi_ptt_pmu_start(event, PERF_EF_RELOAD);
1175 if (hwc->state & PERF_HES_STOPPED)
1176 return -EINVAL;
1177 }
1178
1179 return 0;
1180 }
1181
1182 static void hisi_ptt_pmu_del(struct perf_event *event, int flags)
1183 {
1184 hisi_ptt_pmu_stop(event, PERF_EF_UPDATE);
1185 }
1186
1187 static void hisi_ptt_pmu_read(struct perf_event *event)
1188 {
1189 }
1190
1191 static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node)
1192 {
1193 cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node);
1194 }
1195
1196 static void hisi_ptt_unregister_pmu(void *pmu)
1197 {
1198 perf_pmu_unregister(pmu);
1199 }
1200
1201 static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt)
1202 {
1203 u16 core_id, sicl_id;
1204 char *pmu_name;
1205 u32 reg;
1206 int ret;
1207
1208 ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online,
1209 &hisi_ptt->hotplug_node);
1210 if (ret)
1211 return ret;
1212
1213 ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev,
1214 hisi_ptt_remove_cpuhp_instance,
1215 &hisi_ptt->hotplug_node);
1216 if (ret)
1217 return ret;
1218
1219 mutex_init(&hisi_ptt->tune_lock);
1220 spin_lock_init(&hisi_ptt->pmu_lock);
1221
1222 hisi_ptt->hisi_ptt_pmu = (struct pmu) {
1223 .module = THIS_MODULE,
1224 .parent = &hisi_ptt->pdev->dev,
1225 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_NO_EXCLUDE,
1226 .task_ctx_nr = perf_sw_context,
1227 .attr_groups = hisi_ptt_pmu_groups,
1228 .event_init = hisi_ptt_pmu_event_init,
1229 .setup_aux = hisi_ptt_pmu_setup_aux,
1230 .free_aux = hisi_ptt_pmu_free_aux,
1231 .start = hisi_ptt_pmu_start,
1232 .stop = hisi_ptt_pmu_stop,
1233 .add = hisi_ptt_pmu_add,
1234 .del = hisi_ptt_pmu_del,
1235 .read = hisi_ptt_pmu_read,
1236 };
1237
1238 reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION);
1239 core_id = FIELD_GET(HISI_PTT_CORE_ID, reg);
1240 sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg);
1241
1242 pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u",
1243 sicl_id, core_id);
1244 if (!pmu_name)
1245 return -ENOMEM;
1246
1247 ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1);
1248 if (ret)
1249 return ret;
1250
1251 return devm_add_action_or_reset(&hisi_ptt->pdev->dev,
1252 hisi_ptt_unregister_pmu,
1253 &hisi_ptt->hisi_ptt_pmu);
1254 }
1255
1256 static void hisi_ptt_unregister_filter_update_notifier(void *data)
1257 {
1258 struct hisi_ptt *hisi_ptt = data;
1259
1260 bus_unregister_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);
1261
1262 /* Cancel any work that has been queued */
1263 cancel_delayed_work_sync(&hisi_ptt->work);
1264 }
1265
1266 /* Register the bus notifier for dynamically updating the filter list */
1267 static int hisi_ptt_register_filter_update_notifier(struct hisi_ptt *hisi_ptt)
1268 {
1269 int ret;
1270
1271 hisi_ptt->hisi_ptt_nb.notifier_call = hisi_ptt_notifier_call;
1272 ret = bus_register_notifier(&pci_bus_type, &hisi_ptt->hisi_ptt_nb);
1273 if (ret)
1274 return ret;
1275
1276 return devm_add_action_or_reset(&hisi_ptt->pdev->dev,
1277 hisi_ptt_unregister_filter_update_notifier,
1278 hisi_ptt);
1279 }
1280
1281 /*
1282 * The DMA of PTT trace can only use direct mappings due to some
1283 * hardware restriction. Check whether there is no IOMMU or the
1284 * policy of the IOMMU domain is passthrough, otherwise the trace
1285 * cannot work.
1286 *
1287 * The PTT device is supposed to behind an ARM SMMUv3, which
1288 * should have passthrough the device by a quirk.
1289 */
1290 static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev)
1291 {
1292 struct iommu_domain *iommu_domain;
1293
1294 iommu_domain = iommu_get_domain_for_dev(&pdev->dev);
1295 if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY)
1296 return 0;
1297
1298 return -EOPNOTSUPP;
1299 }
1300
1301 static int hisi_ptt_probe(struct pci_dev *pdev,
1302 const struct pci_device_id *id)
1303 {
1304 struct hisi_ptt *hisi_ptt;
1305 int ret;
1306
1307 ret = hisi_ptt_check_iommu_mapping(pdev);
1308 if (ret) {
1309 pci_err(pdev, "requires direct DMA mappings\n");
1310 return ret;
1311 }
1312
1313 hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL);
1314 if (!hisi_ptt)
1315 return -ENOMEM;
1316
1317 hisi_ptt->pdev = pdev;
1318 pci_set_drvdata(pdev, hisi_ptt);
1319
1320 ret = pcim_enable_device(pdev);
1321 if (ret) {
1322 pci_err(pdev, "failed to enable device, ret = %d\n", ret);
1323 return ret;
1324 }
1325
1326 ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME);
1327 if (ret) {
1328 pci_err(pdev, "failed to remap io memory, ret = %d\n", ret);
1329 return ret;
1330 }
1331
1332 hisi_ptt->iobase = pcim_iomap_table(pdev)[2];
1333
1334 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1335 if (ret) {
1336 pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret);
1337 return ret;
1338 }
1339
1340 pci_set_master(pdev);
1341
1342 ret = hisi_ptt_register_irq(hisi_ptt);
1343 if (ret)
1344 return ret;
1345
1346 ret = hisi_ptt_init_ctrls(hisi_ptt);
1347 if (ret) {
1348 pci_err(pdev, "failed to init controls, ret = %d\n", ret);
1349 return ret;
1350 }
1351
1352 ret = hisi_ptt_register_filter_update_notifier(hisi_ptt);
1353 if (ret)
1354 pci_warn(pdev, "failed to register filter update notifier, ret = %d", ret);
1355
1356 ret = hisi_ptt_register_pmu(hisi_ptt);
1357 if (ret) {
1358 pci_err(pdev, "failed to register PMU device, ret = %d", ret);
1359 return ret;
1360 }
1361
1362 ret = hisi_ptt_init_filter_attributes(hisi_ptt);
1363 if (ret) {
1364 pci_err(pdev, "failed to init sysfs filter attributes, ret = %d", ret);
1365 return ret;
1366 }
1367
1368 return 0;
1369 }
1370
1371 static const struct pci_device_id hisi_ptt_id_tbl[] = {
1372 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) },
1373 { }
1374 };
1375 MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl);
1376
1377 static struct pci_driver hisi_ptt_driver = {
1378 .name = DRV_NAME,
1379 .id_table = hisi_ptt_id_tbl,
1380 .probe = hisi_ptt_probe,
1381 };
1382
1383 static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1384 {
1385 struct hisi_ptt *hisi_ptt;
1386 struct device *dev;
1387 int target, src;
1388
1389 hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node);
1390 src = hisi_ptt->trace_ctrl.on_cpu;
1391 dev = hisi_ptt->hisi_ptt_pmu.dev;
1392
1393 if (!hisi_ptt->trace_ctrl.started || src != cpu)
1394 return 0;
1395
1396 target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu);
1397 if (target >= nr_cpu_ids) {
1398 dev_err(dev, "no available cpu for perf context migration\n");
1399 return 0;
1400 }
1401
1402 perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target);
1403
1404 /*
1405 * Also make sure the interrupt bind to the migrated CPU as well. Warn
1406 * the user on failure here.
1407 */
1408 if (irq_set_affinity(hisi_ptt->trace_irq, cpumask_of(target)))
1409 dev_warn(dev, "failed to set the affinity of trace interrupt\n");
1410
1411 hisi_ptt->trace_ctrl.on_cpu = target;
1412 return 0;
1413 }
1414
1415 static int __init hisi_ptt_init(void)
1416 {
1417 int ret;
1418
1419 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL,
1420 hisi_ptt_cpu_teardown);
1421 if (ret < 0)
1422 return ret;
1423 hisi_ptt_pmu_online = ret;
1424
1425 ret = pci_register_driver(&hisi_ptt_driver);
1426 if (ret)
1427 cpuhp_remove_multi_state(hisi_ptt_pmu_online);
1428
1429 return ret;
1430 }
1431 module_init(hisi_ptt_init);
1432
1433 static void __exit hisi_ptt_exit(void)
1434 {
1435 pci_unregister_driver(&hisi_ptt_driver);
1436 cpuhp_remove_multi_state(hisi_ptt_pmu_online);
1437 }
1438 module_exit(hisi_ptt_exit);
1439
1440 MODULE_LICENSE("GPL");
1441 MODULE_AUTHOR("Yicong Yang <yangyicong@hisilicon.com>");
1442 MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device");
Kernel DRM miscellaneous fixes and cross-tree changes