2 * Copyright(C) 2016 Linaro Limited. All rights reserved.
3 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/circ_buf.h>
19 #include <linux/coresight.h>
20 #include <linux/perf_event.h>
21 #include <linux/slab.h>
22 #include "coresight-priv.h"
23 #include "coresight-tmc.h"
25 static void tmc_etb_enable_hw(struct tmc_drvdata
*drvdata
)
27 CS_UNLOCK(drvdata
->base
);
29 /* Wait for TMCSReady bit to be set */
30 tmc_wait_for_tmcready(drvdata
);
32 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER
, drvdata
->base
+ TMC_MODE
);
33 writel_relaxed(TMC_FFCR_EN_FMT
| TMC_FFCR_EN_TI
|
34 TMC_FFCR_FON_FLIN
| TMC_FFCR_FON_TRIG_EVT
|
35 TMC_FFCR_TRIGON_TRIGIN
,
36 drvdata
->base
+ TMC_FFCR
);
38 writel_relaxed(drvdata
->trigger_cntr
, drvdata
->base
+ TMC_TRG
);
39 tmc_enable_hw(drvdata
);
41 CS_LOCK(drvdata
->base
);
44 static void tmc_etb_dump_hw(struct tmc_drvdata
*drvdata
)
49 u32 read_data
, status
;
53 * Get a hold of the status register and see if a wrap around
56 status
= readl_relaxed(drvdata
->base
+ TMC_STS
);
57 if (status
& TMC_STS_FULL
)
62 barrier
= barrier_pkt
;
64 for (i
= 0; i
< drvdata
->memwidth
; i
++) {
65 read_data
= readl_relaxed(drvdata
->base
+ TMC_RRD
);
66 if (read_data
== 0xFFFFFFFF)
69 if (lost
&& *barrier
) {
74 memcpy(bufp
, &read_data
, 4);
81 static void tmc_etb_disable_hw(struct tmc_drvdata
*drvdata
)
83 CS_UNLOCK(drvdata
->base
);
85 tmc_flush_and_stop(drvdata
);
87 * When operating in sysFS mode the content of the buffer needs to be
88 * read before the TMC is disabled.
90 if (drvdata
->mode
== CS_MODE_SYSFS
)
91 tmc_etb_dump_hw(drvdata
);
92 tmc_disable_hw(drvdata
);
94 CS_LOCK(drvdata
->base
);
97 static void tmc_etf_enable_hw(struct tmc_drvdata
*drvdata
)
99 CS_UNLOCK(drvdata
->base
);
101 /* Wait for TMCSReady bit to be set */
102 tmc_wait_for_tmcready(drvdata
);
104 writel_relaxed(TMC_MODE_HARDWARE_FIFO
, drvdata
->base
+ TMC_MODE
);
105 writel_relaxed(TMC_FFCR_EN_FMT
| TMC_FFCR_EN_TI
,
106 drvdata
->base
+ TMC_FFCR
);
107 writel_relaxed(0x0, drvdata
->base
+ TMC_BUFWM
);
108 tmc_enable_hw(drvdata
);
110 CS_LOCK(drvdata
->base
);
113 static void tmc_etf_disable_hw(struct tmc_drvdata
*drvdata
)
115 CS_UNLOCK(drvdata
->base
);
117 tmc_flush_and_stop(drvdata
);
118 tmc_disable_hw(drvdata
);
120 CS_LOCK(drvdata
->base
);
123 static int tmc_enable_etf_sink_sysfs(struct coresight_device
*csdev
)
129 struct tmc_drvdata
*drvdata
= dev_get_drvdata(csdev
->dev
.parent
);
132 * If we don't have a buffer release the lock and allocate memory.
133 * Otherwise keep the lock and move along.
135 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
137 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
139 /* Allocating the memory here while outside of the spinlock */
140 buf
= kzalloc(drvdata
->size
, GFP_KERNEL
);
144 /* Let's try again */
145 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
148 if (drvdata
->reading
) {
154 * In sysFS mode we can have multiple writers per sink. Since this
155 * sink is already enabled no memory is needed and the HW need not be
158 if (drvdata
->mode
== CS_MODE_SYSFS
)
162 * If drvdata::buf isn't NULL, memory was allocated for a previous
163 * trace run but wasn't read. If so simply zero-out the memory.
164 * Otherwise use the memory allocated above.
166 * The memory is freed when users read the buffer using the
167 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for
171 memset(drvdata
->buf
, 0, drvdata
->size
);
177 drvdata
->mode
= CS_MODE_SYSFS
;
178 tmc_etb_enable_hw(drvdata
);
180 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
182 /* Free memory outside the spinlock if need be */
189 static int tmc_enable_etf_sink_perf(struct coresight_device
*csdev
)
193 struct tmc_drvdata
*drvdata
= dev_get_drvdata(csdev
->dev
.parent
);
195 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
196 if (drvdata
->reading
) {
202 * In Perf mode there can be only one writer per sink. There
203 * is also no need to continue if the ETB/ETR is already operated
206 if (drvdata
->mode
!= CS_MODE_DISABLED
) {
211 drvdata
->mode
= CS_MODE_PERF
;
212 tmc_etb_enable_hw(drvdata
);
214 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
219 static int tmc_enable_etf_sink(struct coresight_device
*csdev
, u32 mode
)
222 struct tmc_drvdata
*drvdata
= dev_get_drvdata(csdev
->dev
.parent
);
226 ret
= tmc_enable_etf_sink_sysfs(csdev
);
229 ret
= tmc_enable_etf_sink_perf(csdev
);
231 /* We shouldn't be here */
240 dev_info(drvdata
->dev
, "TMC-ETB/ETF enabled\n");
244 static void tmc_disable_etf_sink(struct coresight_device
*csdev
)
247 struct tmc_drvdata
*drvdata
= dev_get_drvdata(csdev
->dev
.parent
);
249 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
250 if (drvdata
->reading
) {
251 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
255 /* Disable the TMC only if it needs to */
256 if (drvdata
->mode
!= CS_MODE_DISABLED
) {
257 tmc_etb_disable_hw(drvdata
);
258 drvdata
->mode
= CS_MODE_DISABLED
;
261 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
263 dev_info(drvdata
->dev
, "TMC-ETB/ETF disabled\n");
266 static int tmc_enable_etf_link(struct coresight_device
*csdev
,
267 int inport
, int outport
)
270 struct tmc_drvdata
*drvdata
= dev_get_drvdata(csdev
->dev
.parent
);
272 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
273 if (drvdata
->reading
) {
274 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
278 tmc_etf_enable_hw(drvdata
);
279 drvdata
->mode
= CS_MODE_SYSFS
;
280 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
282 dev_info(drvdata
->dev
, "TMC-ETF enabled\n");
286 static void tmc_disable_etf_link(struct coresight_device
*csdev
,
287 int inport
, int outport
)
290 struct tmc_drvdata
*drvdata
= dev_get_drvdata(csdev
->dev
.parent
);
292 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
293 if (drvdata
->reading
) {
294 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
298 tmc_etf_disable_hw(drvdata
);
299 drvdata
->mode
= CS_MODE_DISABLED
;
300 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
302 dev_info(drvdata
->dev
, "TMC-ETF disabled\n");
305 static void *tmc_alloc_etf_buffer(struct coresight_device
*csdev
, int cpu
,
306 void **pages
, int nr_pages
, bool overwrite
)
309 struct cs_buffers
*buf
;
312 cpu
= smp_processor_id();
313 node
= cpu_to_node(cpu
);
315 /* Allocate memory structure for interaction with Perf */
316 buf
= kzalloc_node(sizeof(struct cs_buffers
), GFP_KERNEL
, node
);
320 buf
->snapshot
= overwrite
;
321 buf
->nr_pages
= nr_pages
;
322 buf
->data_pages
= pages
;
327 static void tmc_free_etf_buffer(void *config
)
329 struct cs_buffers
*buf
= config
;
334 static int tmc_set_etf_buffer(struct coresight_device
*csdev
,
335 struct perf_output_handle
*handle
,
340 struct cs_buffers
*buf
= sink_config
;
342 /* wrap head around to the amount of space we have */
343 head
= handle
->head
& ((buf
->nr_pages
<< PAGE_SHIFT
) - 1);
345 /* find the page to write to */
346 buf
->cur
= head
/ PAGE_SIZE
;
348 /* and offset within that page */
349 buf
->offset
= head
% PAGE_SIZE
;
351 local_set(&buf
->data_size
, 0);
356 static unsigned long tmc_reset_etf_buffer(struct coresight_device
*csdev
,
357 struct perf_output_handle
*handle
,
361 struct cs_buffers
*buf
= sink_config
;
365 * In snapshot mode ->data_size holds the new address of the
366 * ring buffer's head. The size itself is the whole address
367 * range since we want the latest information.
370 handle
->head
= local_xchg(&buf
->data_size
,
371 buf
->nr_pages
<< PAGE_SHIFT
);
373 * Tell the tracer PMU how much we got in this run and if
374 * something went wrong along the way. Nobody else can use
375 * this cs_buffers instance until we are done. As such
376 * resetting parameters here and squaring off with the ring
377 * buffer API in the tracer PMU is fine.
379 size
= local_xchg(&buf
->data_size
, 0);
385 static void tmc_update_etf_buffer(struct coresight_device
*csdev
,
386 struct perf_output_handle
*handle
,
393 u64 read_ptr
, write_ptr
;
395 unsigned long offset
;
396 struct cs_buffers
*buf
= sink_config
;
397 struct tmc_drvdata
*drvdata
= dev_get_drvdata(csdev
->dev
.parent
);
402 /* This shouldn't happen */
403 if (WARN_ON_ONCE(drvdata
->mode
!= CS_MODE_PERF
))
406 CS_UNLOCK(drvdata
->base
);
408 tmc_flush_and_stop(drvdata
);
410 read_ptr
= tmc_read_rrp(drvdata
);
411 write_ptr
= tmc_read_rwp(drvdata
);
414 * Get a hold of the status register and see if a wrap around
415 * has occurred. If so adjust things accordingly.
417 status
= readl_relaxed(drvdata
->base
+ TMC_STS
);
418 if (status
& TMC_STS_FULL
) {
420 to_read
= drvdata
->size
;
422 to_read
= CIRC_CNT(write_ptr
, read_ptr
, drvdata
->size
);
426 * The TMC RAM buffer may be bigger than the space available in the
427 * perf ring buffer (handle->size). If so advance the RRP so that we
428 * get the latest trace data.
430 if (to_read
> handle
->size
) {
434 * The value written to RRP must be byte-address aligned to
435 * the width of the trace memory databus _and_ to a frame
436 * boundary (16 byte), whichever is the biggest. For example,
437 * for 32-bit, 64-bit and 128-bit wide trace memory, the four
438 * LSBs must be 0s. For 256-bit wide trace memory, the five
441 switch (drvdata
->memwidth
) {
442 case TMC_MEM_INTF_WIDTH_32BITS
:
443 case TMC_MEM_INTF_WIDTH_64BITS
:
444 case TMC_MEM_INTF_WIDTH_128BITS
:
445 mask
= GENMASK(31, 5);
447 case TMC_MEM_INTF_WIDTH_256BITS
:
448 mask
= GENMASK(31, 6);
453 * Make sure the new size is aligned in accordance with the
454 * requirement explained above.
456 to_read
= handle
->size
& mask
;
457 /* Move the RAM read pointer up */
458 read_ptr
= (write_ptr
+ drvdata
->size
) - to_read
;
459 /* Make sure we are still within our limits */
460 if (read_ptr
> (drvdata
->size
- 1))
461 read_ptr
-= drvdata
->size
;
463 tmc_write_rrp(drvdata
, read_ptr
);
468 perf_aux_output_flag(handle
, PERF_AUX_FLAG_TRUNCATED
);
471 offset
= buf
->offset
;
472 barrier
= barrier_pkt
;
474 /* for every byte to read */
475 for (i
= 0; i
< to_read
; i
+= 4) {
476 buf_ptr
= buf
->data_pages
[cur
] + offset
;
477 *buf_ptr
= readl_relaxed(drvdata
->base
+ TMC_RRD
);
479 if (lost
&& *barrier
) {
485 if (offset
>= PAGE_SIZE
) {
488 /* wrap around at the end of the buffer */
489 cur
&= buf
->nr_pages
- 1;
494 * In snapshot mode all we have to do is communicate to
495 * perf_aux_output_end() the address of the current head. In full
496 * trace mode the same function expects a size to move rb->aux_head
500 local_set(&buf
->data_size
, (cur
* PAGE_SIZE
) + offset
);
502 local_add(to_read
, &buf
->data_size
);
504 CS_LOCK(drvdata
->base
);
507 static const struct coresight_ops_sink tmc_etf_sink_ops
= {
508 .enable
= tmc_enable_etf_sink
,
509 .disable
= tmc_disable_etf_sink
,
510 .alloc_buffer
= tmc_alloc_etf_buffer
,
511 .free_buffer
= tmc_free_etf_buffer
,
512 .set_buffer
= tmc_set_etf_buffer
,
513 .reset_buffer
= tmc_reset_etf_buffer
,
514 .update_buffer
= tmc_update_etf_buffer
,
517 static const struct coresight_ops_link tmc_etf_link_ops
= {
518 .enable
= tmc_enable_etf_link
,
519 .disable
= tmc_disable_etf_link
,
522 const struct coresight_ops tmc_etb_cs_ops
= {
523 .sink_ops
= &tmc_etf_sink_ops
,
526 const struct coresight_ops tmc_etf_cs_ops
= {
527 .sink_ops
= &tmc_etf_sink_ops
,
528 .link_ops
= &tmc_etf_link_ops
,
531 int tmc_read_prepare_etb(struct tmc_drvdata
*drvdata
)
537 /* config types are set a boot time and never change */
538 if (WARN_ON_ONCE(drvdata
->config_type
!= TMC_CONFIG_TYPE_ETB
&&
539 drvdata
->config_type
!= TMC_CONFIG_TYPE_ETF
))
542 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
544 if (drvdata
->reading
) {
549 /* There is no point in reading a TMC in HW FIFO mode */
550 mode
= readl_relaxed(drvdata
->base
+ TMC_MODE
);
551 if (mode
!= TMC_MODE_CIRCULAR_BUFFER
) {
556 /* Don't interfere if operated from Perf */
557 if (drvdata
->mode
== CS_MODE_PERF
) {
562 /* If drvdata::buf is NULL the trace data has been read already */
563 if (drvdata
->buf
== NULL
) {
568 /* Disable the TMC if need be */
569 if (drvdata
->mode
== CS_MODE_SYSFS
)
570 tmc_etb_disable_hw(drvdata
);
572 drvdata
->reading
= true;
574 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
579 int tmc_read_unprepare_etb(struct tmc_drvdata
*drvdata
)
585 /* config types are set a boot time and never change */
586 if (WARN_ON_ONCE(drvdata
->config_type
!= TMC_CONFIG_TYPE_ETB
&&
587 drvdata
->config_type
!= TMC_CONFIG_TYPE_ETF
))
590 spin_lock_irqsave(&drvdata
->spinlock
, flags
);
592 /* There is no point in reading a TMC in HW FIFO mode */
593 mode
= readl_relaxed(drvdata
->base
+ TMC_MODE
);
594 if (mode
!= TMC_MODE_CIRCULAR_BUFFER
) {
595 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
599 /* Re-enable the TMC if need be */
600 if (drvdata
->mode
== CS_MODE_SYSFS
) {
602 * The trace run will continue with the same allocated trace
603 * buffer. As such zero-out the buffer so that we don't end
604 * up with stale data.
606 * Since the tracer is still enabled drvdata::buf
609 memset(drvdata
->buf
, 0, drvdata
->size
);
610 tmc_etb_enable_hw(drvdata
);
613 * The ETB/ETF is not tracing and the buffer was just read.
614 * As such prepare to free the trace buffer.
620 drvdata
->reading
= false;
621 spin_unlock_irqrestore(&drvdata
->spinlock
, flags
);
624 * Free allocated memory outside of the spinlock. There is no need
625 * to assert the validity of 'buf' since calling kfree(NULL) is safe.