?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dt-bindings: pinctrl: add bindings for MediaTek MT6779 SoC
[linux/fpc-iii.git] / kernel / trace / trace_events_hist.c
blob0b933546142e8b371bf42550d3c0026dfdd5128b
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * trace_events_hist - trace event hist triggers
4 *
5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
6 */
7
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/security.h>
11 #include <linux/mutex.h>
12 #include <linux/slab.h>
13 #include <linux/stacktrace.h>
14 #include <linux/rculist.h>
15 #include <linux/tracefs.h>
16
17 /* for gfp flag names */
18 #include <linux/trace_events.h>
19 #include <trace/events/mmflags.h>
20
21 #include "tracing_map.h"
22 #include "trace_synth.h"
23
24 #define ERRORS \
25 C(NONE, "No error"), \
26 C(DUPLICATE_VAR, "Variable already defined"), \
27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28 C(TOO_MANY_VARS, "Too many variables defined"), \
29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \
30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \
31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \
32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \
34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \
35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \
36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \
37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \
38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \
39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \
40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \
41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \
42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \
43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \
44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \
46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \
47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \
48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \
49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \
50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \
51 C(TOO_MANY_PARAMS, "Too many action params"), \
52 C(PARAM_NOT_FOUND, "Couldn't find param"), \
53 C(INVALID_PARAM, "Invalid action param"), \
54 C(ACTION_NOT_FOUND, "No action found"), \
55 C(NO_SAVE_PARAMS, "No params found for save()"), \
56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57 C(ACTION_MISMATCH, "Handler doesn't support action"), \
58 C(NO_CLOSING_PAREN, "No closing paren found"), \
59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \
60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \
61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \
62 C(VAR_NOT_FOUND, "Couldn't find variable"), \
63 C(FIELD_NOT_FOUND, "Couldn't find field"), \
64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \
65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \
66 C(EMPTY_SORT_FIELD, "Empty sort field"), \
67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \
68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"),
69
70 #undef C
71 #define C(a, b) HIST_ERR_##a
72
73 enum { ERRORS };
74
75 #undef C
76 #define C(a, b) b
77
78 static const char *err_text[] = { ERRORS };
79
80 struct hist_field;
81
82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
83 struct tracing_map_elt *elt,
84 struct ring_buffer_event *rbe,
85 void *event);
86
87 #define HIST_FIELD_OPERANDS_MAX 2
88 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
89 #define HIST_ACTIONS_MAX 8
90
91 enum field_op_id {
92 FIELD_OP_NONE,
93 FIELD_OP_PLUS,
94 FIELD_OP_MINUS,
95 FIELD_OP_UNARY_MINUS,
96 };
97
98 /*
99 * A hist_var (histogram variable) contains variable information for
100 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
101 * flag set. A hist_var has a variable name e.g. ts0, and is
102 * associated with a given histogram trigger, as specified by
103 * hist_data. The hist_var idx is the unique index assigned to the
104 * variable by the hist trigger's tracing_map. The idx is what is
105 * used to set a variable's value and, by a variable reference, to
106 * retrieve it.
107 */
108 struct hist_var {
109 char *name;
110 struct hist_trigger_data *hist_data;
111 unsigned int idx;
112 };
113
114 struct hist_field {
115 struct ftrace_event_field *field;
116 unsigned long flags;
117 hist_field_fn_t fn;
118 unsigned int ref;
119 unsigned int size;
120 unsigned int offset;
121 unsigned int is_signed;
122 const char *type;
123 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
124 struct hist_trigger_data *hist_data;
125
126 /*
127 * Variable fields contain variable-specific info in var.
128 */
129 struct hist_var var;
130 enum field_op_id operator;
131 char *system;
132 char *event_name;
133
134 /*
135 * The name field is used for EXPR and VAR_REF fields. VAR
136 * fields contain the variable name in var.name.
137 */
138 char *name;
139
140 /*
141 * When a histogram trigger is hit, if it has any references
142 * to variables, the values of those variables are collected
143 * into a var_ref_vals array by resolve_var_refs(). The
144 * current value of each variable is read from the tracing_map
145 * using the hist field's hist_var.idx and entered into the
146 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
147 */
148 unsigned int var_ref_idx;
149 bool read_once;
150 };
151
152 static u64 hist_field_none(struct hist_field *field,
153 struct tracing_map_elt *elt,
154 struct ring_buffer_event *rbe,
155 void *event)
156 {
157 return 0;
158 }
159
160 static u64 hist_field_counter(struct hist_field *field,
161 struct tracing_map_elt *elt,
162 struct ring_buffer_event *rbe,
163 void *event)
164 {
165 return 1;
166 }
167
168 static u64 hist_field_string(struct hist_field *hist_field,
169 struct tracing_map_elt *elt,
170 struct ring_buffer_event *rbe,
171 void *event)
172 {
173 char *addr = (char *)(event + hist_field->field->offset);
174
175 return (u64)(unsigned long)addr;
176 }
177
178 static u64 hist_field_dynstring(struct hist_field *hist_field,
179 struct tracing_map_elt *elt,
180 struct ring_buffer_event *rbe,
181 void *event)
182 {
183 u32 str_item = *(u32 *)(event + hist_field->field->offset);
184 int str_loc = str_item & 0xffff;
185 char *addr = (char *)(event + str_loc);
186
187 return (u64)(unsigned long)addr;
188 }
189
190 static u64 hist_field_pstring(struct hist_field *hist_field,
191 struct tracing_map_elt *elt,
192 struct ring_buffer_event *rbe,
193 void *event)
194 {
195 char **addr = (char **)(event + hist_field->field->offset);
196
197 return (u64)(unsigned long)*addr;
198 }
199
200 static u64 hist_field_log2(struct hist_field *hist_field,
201 struct tracing_map_elt *elt,
202 struct ring_buffer_event *rbe,
203 void *event)
204 {
205 struct hist_field *operand = hist_field->operands[0];
206
207 u64 val = operand->fn(operand, elt, rbe, event);
208
209 return (u64) ilog2(roundup_pow_of_two(val));
210 }
211
212 static u64 hist_field_plus(struct hist_field *hist_field,
213 struct tracing_map_elt *elt,
214 struct ring_buffer_event *rbe,
215 void *event)
216 {
217 struct hist_field *operand1 = hist_field->operands[0];
218 struct hist_field *operand2 = hist_field->operands[1];
219
220 u64 val1 = operand1->fn(operand1, elt, rbe, event);
221 u64 val2 = operand2->fn(operand2, elt, rbe, event);
222
223 return val1 + val2;
224 }
225
226 static u64 hist_field_minus(struct hist_field *hist_field,
227 struct tracing_map_elt *elt,
228 struct ring_buffer_event *rbe,
229 void *event)
230 {
231 struct hist_field *operand1 = hist_field->operands[0];
232 struct hist_field *operand2 = hist_field->operands[1];
233
234 u64 val1 = operand1->fn(operand1, elt, rbe, event);
235 u64 val2 = operand2->fn(operand2, elt, rbe, event);
236
237 return val1 - val2;
238 }
239
240 static u64 hist_field_unary_minus(struct hist_field *hist_field,
241 struct tracing_map_elt *elt,
242 struct ring_buffer_event *rbe,
243 void *event)
244 {
245 struct hist_field *operand = hist_field->operands[0];
246
247 s64 sval = (s64)operand->fn(operand, elt, rbe, event);
248 u64 val = (u64)-sval;
249
250 return val;
251 }
252
253 #define DEFINE_HIST_FIELD_FN(type) \
254 static u64 hist_field_##type(struct hist_field *hist_field, \
255 struct tracing_map_elt *elt, \
256 struct ring_buffer_event *rbe, \
257 void *event) \
258 { \
259 type *addr = (type *)(event + hist_field->field->offset); \
260 \
261 return (u64)(unsigned long)*addr; \
262 }
263
264 DEFINE_HIST_FIELD_FN(s64);
265 DEFINE_HIST_FIELD_FN(u64);
266 DEFINE_HIST_FIELD_FN(s32);
267 DEFINE_HIST_FIELD_FN(u32);
268 DEFINE_HIST_FIELD_FN(s16);
269 DEFINE_HIST_FIELD_FN(u16);
270 DEFINE_HIST_FIELD_FN(s8);
271 DEFINE_HIST_FIELD_FN(u8);
272
273 #define for_each_hist_field(i, hist_data) \
274 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
275
276 #define for_each_hist_val_field(i, hist_data) \
277 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
278
279 #define for_each_hist_key_field(i, hist_data) \
280 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
281
282 #define HIST_STACKTRACE_DEPTH 16
283 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
284 #define HIST_STACKTRACE_SKIP 5
285
286 #define HITCOUNT_IDX 0
287 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
288
289 enum hist_field_flags {
290 HIST_FIELD_FL_HITCOUNT = 1 << 0,
291 HIST_FIELD_FL_KEY = 1 << 1,
292 HIST_FIELD_FL_STRING = 1 << 2,
293 HIST_FIELD_FL_HEX = 1 << 3,
294 HIST_FIELD_FL_SYM = 1 << 4,
295 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
296 HIST_FIELD_FL_EXECNAME = 1 << 6,
297 HIST_FIELD_FL_SYSCALL = 1 << 7,
298 HIST_FIELD_FL_STACKTRACE = 1 << 8,
299 HIST_FIELD_FL_LOG2 = 1 << 9,
300 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
301 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
302 HIST_FIELD_FL_VAR = 1 << 12,
303 HIST_FIELD_FL_EXPR = 1 << 13,
304 HIST_FIELD_FL_VAR_REF = 1 << 14,
305 HIST_FIELD_FL_CPU = 1 << 15,
306 HIST_FIELD_FL_ALIAS = 1 << 16,
307 };
308
309 struct var_defs {
310 unsigned int n_vars;
311 char *name[TRACING_MAP_VARS_MAX];
312 char *expr[TRACING_MAP_VARS_MAX];
313 };
314
315 struct hist_trigger_attrs {
316 char *keys_str;
317 char *vals_str;
318 char *sort_key_str;
319 char *name;
320 char *clock;
321 bool pause;
322 bool cont;
323 bool clear;
324 bool ts_in_usecs;
325 unsigned int map_bits;
326
327 char *assignment_str[TRACING_MAP_VARS_MAX];
328 unsigned int n_assignments;
329
330 char *action_str[HIST_ACTIONS_MAX];
331 unsigned int n_actions;
332
333 struct var_defs var_defs;
334 };
335
336 struct field_var {
337 struct hist_field *var;
338 struct hist_field *val;
339 };
340
341 struct field_var_hist {
342 struct hist_trigger_data *hist_data;
343 char *cmd;
344 };
345
346 struct hist_trigger_data {
347 struct hist_field *fields[HIST_FIELDS_MAX];
348 unsigned int n_vals;
349 unsigned int n_keys;
350 unsigned int n_fields;
351 unsigned int n_vars;
352 unsigned int key_size;
353 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
354 unsigned int n_sort_keys;
355 struct trace_event_file *event_file;
356 struct hist_trigger_attrs *attrs;
357 struct tracing_map *map;
358 bool enable_timestamps;
359 bool remove;
360 struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
361 unsigned int n_var_refs;
362
363 struct action_data *actions[HIST_ACTIONS_MAX];
364 unsigned int n_actions;
365
366 struct field_var *field_vars[SYNTH_FIELDS_MAX];
367 unsigned int n_field_vars;
368 unsigned int n_field_var_str;
369 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
370 unsigned int n_field_var_hists;
371
372 struct field_var *save_vars[SYNTH_FIELDS_MAX];
373 unsigned int n_save_vars;
374 unsigned int n_save_var_str;
375 };
376
377 struct action_data;
378
379 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
380 struct tracing_map_elt *elt, void *rec,
381 struct ring_buffer_event *rbe, void *key,
382 struct action_data *data, u64 *var_ref_vals);
383
384 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
385
386 enum handler_id {
387 HANDLER_ONMATCH = 1,
388 HANDLER_ONMAX,
389 HANDLER_ONCHANGE,
390 };
391
392 enum action_id {
393 ACTION_SAVE = 1,
394 ACTION_TRACE,
395 ACTION_SNAPSHOT,
396 };
397
398 struct action_data {
399 enum handler_id handler;
400 enum action_id action;
401 char *action_name;
402 action_fn_t fn;
403
404 unsigned int n_params;
405 char *params[SYNTH_FIELDS_MAX];
406
407 /*
408 * When a histogram trigger is hit, the values of any
409 * references to variables, including variables being passed
410 * as parameters to synthetic events, are collected into a
411 * var_ref_vals array. This var_ref_idx array is an array of
412 * indices into the var_ref_vals array, one for each synthetic
413 * event param, and is passed to the synthetic event
414 * invocation.
415 */
416 unsigned int var_ref_idx[TRACING_MAP_VARS_MAX];
417 struct synth_event *synth_event;
418 bool use_trace_keyword;
419 char *synth_event_name;
420
421 union {
422 struct {
423 char *event;
424 char *event_system;
425 } match_data;
426
427 struct {
428 /*
429 * var_str contains the $-unstripped variable
430 * name referenced by var_ref, and used when
431 * printing the action. Because var_ref
432 * creation is deferred to create_actions(),
433 * we need a per-action way to save it until
434 * then, thus var_str.
435 */
436 char *var_str;
437
438 /*
439 * var_ref refers to the variable being
440 * tracked e.g onmax($var).
441 */
442 struct hist_field *var_ref;
443
444 /*
445 * track_var contains the 'invisible' tracking
446 * variable created to keep the current
447 * e.g. max value.
448 */
449 struct hist_field *track_var;
450
451 check_track_val_fn_t check_val;
452 action_fn_t save_data;
453 } track_data;
454 };
455 };
456
457 struct track_data {
458 u64 track_val;
459 bool updated;
460
461 unsigned int key_len;
462 void *key;
463 struct tracing_map_elt elt;
464
465 struct action_data *action_data;
466 struct hist_trigger_data *hist_data;
467 };
468
469 struct hist_elt_data {
470 char *comm;
471 u64 *var_ref_vals;
472 char *field_var_str[SYNTH_FIELDS_MAX];
473 };
474
475 struct snapshot_context {
476 struct tracing_map_elt *elt;
477 void *key;
478 };
479
480 static void track_data_free(struct track_data *track_data)
481 {
482 struct hist_elt_data *elt_data;
483
484 if (!track_data)
485 return;
486
487 kfree(track_data->key);
488
489 elt_data = track_data->elt.private_data;
490 if (elt_data) {
491 kfree(elt_data->comm);
492 kfree(elt_data);
493 }
494
495 kfree(track_data);
496 }
497
498 static struct track_data *track_data_alloc(unsigned int key_len,
499 struct action_data *action_data,
500 struct hist_trigger_data *hist_data)
501 {
502 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
503 struct hist_elt_data *elt_data;
504
505 if (!data)
506 return ERR_PTR(-ENOMEM);
507
508 data->key = kzalloc(key_len, GFP_KERNEL);
509 if (!data->key) {
510 track_data_free(data);
511 return ERR_PTR(-ENOMEM);
512 }
513
514 data->key_len = key_len;
515 data->action_data = action_data;
516 data->hist_data = hist_data;
517
518 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
519 if (!elt_data) {
520 track_data_free(data);
521 return ERR_PTR(-ENOMEM);
522 }
523
524 data->elt.private_data = elt_data;
525
526 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
527 if (!elt_data->comm) {
528 track_data_free(data);
529 return ERR_PTR(-ENOMEM);
530 }
531
532 return data;
533 }
534
535 static char last_cmd[MAX_FILTER_STR_VAL];
536 static char last_cmd_loc[MAX_FILTER_STR_VAL];
537
538 static int errpos(char *str)
539 {
540 return err_pos(last_cmd, str);
541 }
542
543 static void last_cmd_set(struct trace_event_file *file, char *str)
544 {
545 const char *system = NULL, *name = NULL;
546 struct trace_event_call *call;
547
548 if (!str)
549 return;
550
551 strcpy(last_cmd, "hist:");
552 strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
553
554 if (file) {
555 call = file->event_call;
556 system = call->class->system;
557 if (system) {
558 name = trace_event_name(call);
559 if (!name)
560 system = NULL;
561 }
562 }
563
564 if (system)
565 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
566 }
567
568 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
569 {
570 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
571 err_type, err_pos);
572 }
573
574 static void hist_err_clear(void)
575 {
576 last_cmd[0] = '\0';
577 last_cmd_loc[0] = '\0';
578 }
579
580 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
581 unsigned int *var_ref_idx);
582
583 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
584 unsigned int *var_ref_idx)
585 {
586 struct tracepoint *tp = event->tp;
587
588 if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
589 struct tracepoint_func *probe_func_ptr;
590 synth_probe_func_t probe_func;
591 void *__data;
592
593 if (!(cpu_online(raw_smp_processor_id())))
594 return;
595
596 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
597 if (probe_func_ptr) {
598 do {
599 probe_func = probe_func_ptr->func;
600 __data = probe_func_ptr->data;
601 probe_func(__data, var_ref_vals, var_ref_idx);
602 } while ((++probe_func_ptr)->func);
603 }
604 }
605 }
606
607 static void action_trace(struct hist_trigger_data *hist_data,
608 struct tracing_map_elt *elt, void *rec,
609 struct ring_buffer_event *rbe, void *key,
610 struct action_data *data, u64 *var_ref_vals)
611 {
612 struct synth_event *event = data->synth_event;
613
614 trace_synth(event, var_ref_vals, data->var_ref_idx);
615 }
616
617 struct hist_var_data {
618 struct list_head list;
619 struct hist_trigger_data *hist_data;
620 };
621
622 static u64 hist_field_timestamp(struct hist_field *hist_field,
623 struct tracing_map_elt *elt,
624 struct ring_buffer_event *rbe,
625 void *event)
626 {
627 struct hist_trigger_data *hist_data = hist_field->hist_data;
628 struct trace_array *tr = hist_data->event_file->tr;
629
630 u64 ts = ring_buffer_event_time_stamp(rbe);
631
632 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
633 ts = ns2usecs(ts);
634
635 return ts;
636 }
637
638 static u64 hist_field_cpu(struct hist_field *hist_field,
639 struct tracing_map_elt *elt,
640 struct ring_buffer_event *rbe,
641 void *event)
642 {
643 int cpu = smp_processor_id();
644
645 return cpu;
646 }
647
648 /**
649 * check_field_for_var_ref - Check if a VAR_REF field references a variable
650 * @hist_field: The VAR_REF field to check
651 * @var_data: The hist trigger that owns the variable
652 * @var_idx: The trigger variable identifier
653 *
654 * Check the given VAR_REF field to see whether or not it references
655 * the given variable associated with the given trigger.
656 *
657 * Return: The VAR_REF field if it does reference the variable, NULL if not
658 */
659 static struct hist_field *
660 check_field_for_var_ref(struct hist_field *hist_field,
661 struct hist_trigger_data *var_data,
662 unsigned int var_idx)
663 {
664 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
665
666 if (hist_field && hist_field->var.idx == var_idx &&
667 hist_field->var.hist_data == var_data)
668 return hist_field;
669
670 return NULL;
671 }
672
673 /**
674 * find_var_ref - Check if a trigger has a reference to a trigger variable
675 * @hist_data: The hist trigger that might have a reference to the variable
676 * @var_data: The hist trigger that owns the variable
677 * @var_idx: The trigger variable identifier
678 *
679 * Check the list of var_refs[] on the first hist trigger to see
680 * whether any of them are references to the variable on the second
681 * trigger.
682 *
683 * Return: The VAR_REF field referencing the variable if so, NULL if not
684 */
685 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
686 struct hist_trigger_data *var_data,
687 unsigned int var_idx)
688 {
689 struct hist_field *hist_field;
690 unsigned int i;
691
692 for (i = 0; i < hist_data->n_var_refs; i++) {
693 hist_field = hist_data->var_refs[i];
694 if (check_field_for_var_ref(hist_field, var_data, var_idx))
695 return hist_field;
696 }
697
698 return NULL;
699 }
700
701 /**
702 * find_any_var_ref - Check if there is a reference to a given trigger variable
703 * @hist_data: The hist trigger
704 * @var_idx: The trigger variable identifier
705 *
706 * Check to see whether the given variable is currently referenced by
707 * any other trigger.
708 *
709 * The trigger the variable is defined on is explicitly excluded - the
710 * assumption being that a self-reference doesn't prevent a trigger
711 * from being removed.
712 *
713 * Return: The VAR_REF field referencing the variable if so, NULL if not
714 */
715 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
716 unsigned int var_idx)
717 {
718 struct trace_array *tr = hist_data->event_file->tr;
719 struct hist_field *found = NULL;
720 struct hist_var_data *var_data;
721
722 list_for_each_entry(var_data, &tr->hist_vars, list) {
723 if (var_data->hist_data == hist_data)
724 continue;
725 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
726 if (found)
727 break;
728 }
729
730 return found;
731 }
732
733 /**
734 * check_var_refs - Check if there is a reference to any of trigger's variables
735 * @hist_data: The hist trigger
736 *
737 * A trigger can define one or more variables. If any one of them is
738 * currently referenced by any other trigger, this function will
739 * determine that.
740
741 * Typically used to determine whether or not a trigger can be removed
742 * - if there are any references to a trigger's variables, it cannot.
743 *
744 * Return: True if there is a reference to any of trigger's variables
745 */
746 static bool check_var_refs(struct hist_trigger_data *hist_data)
747 {
748 struct hist_field *field;
749 bool found = false;
750 int i;
751
752 for_each_hist_field(i, hist_data) {
753 field = hist_data->fields[i];
754 if (field && field->flags & HIST_FIELD_FL_VAR) {
755 if (find_any_var_ref(hist_data, field->var.idx)) {
756 found = true;
757 break;
758 }
759 }
760 }
761
762 return found;
763 }
764
765 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
766 {
767 struct trace_array *tr = hist_data->event_file->tr;
768 struct hist_var_data *var_data, *found = NULL;
769
770 list_for_each_entry(var_data, &tr->hist_vars, list) {
771 if (var_data->hist_data == hist_data) {
772 found = var_data;
773 break;
774 }
775 }
776
777 return found;
778 }
779
780 static bool field_has_hist_vars(struct hist_field *hist_field,
781 unsigned int level)
782 {
783 int i;
784
785 if (level > 3)
786 return false;
787
788 if (!hist_field)
789 return false;
790
791 if (hist_field->flags & HIST_FIELD_FL_VAR ||
792 hist_field->flags & HIST_FIELD_FL_VAR_REF)
793 return true;
794
795 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
796 struct hist_field *operand;
797
798 operand = hist_field->operands[i];
799 if (field_has_hist_vars(operand, level + 1))
800 return true;
801 }
802
803 return false;
804 }
805
806 static bool has_hist_vars(struct hist_trigger_data *hist_data)
807 {
808 struct hist_field *hist_field;
809 int i;
810
811 for_each_hist_field(i, hist_data) {
812 hist_field = hist_data->fields[i];
813 if (field_has_hist_vars(hist_field, 0))
814 return true;
815 }
816
817 return false;
818 }
819
820 static int save_hist_vars(struct hist_trigger_data *hist_data)
821 {
822 struct trace_array *tr = hist_data->event_file->tr;
823 struct hist_var_data *var_data;
824
825 var_data = find_hist_vars(hist_data);
826 if (var_data)
827 return 0;
828
829 if (tracing_check_open_get_tr(tr))
830 return -ENODEV;
831
832 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
833 if (!var_data) {
834 trace_array_put(tr);
835 return -ENOMEM;
836 }
837
838 var_data->hist_data = hist_data;
839 list_add(&var_data->list, &tr->hist_vars);
840
841 return 0;
842 }
843
844 static void remove_hist_vars(struct hist_trigger_data *hist_data)
845 {
846 struct trace_array *tr = hist_data->event_file->tr;
847 struct hist_var_data *var_data;
848
849 var_data = find_hist_vars(hist_data);
850 if (!var_data)
851 return;
852
853 if (WARN_ON(check_var_refs(hist_data)))
854 return;
855
856 list_del(&var_data->list);
857
858 kfree(var_data);
859
860 trace_array_put(tr);
861 }
862
863 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
864 const char *var_name)
865 {
866 struct hist_field *hist_field, *found = NULL;
867 int i;
868
869 for_each_hist_field(i, hist_data) {
870 hist_field = hist_data->fields[i];
871 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
872 strcmp(hist_field->var.name, var_name) == 0) {
873 found = hist_field;
874 break;
875 }
876 }
877
878 return found;
879 }
880
881 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
882 struct trace_event_file *file,
883 const char *var_name)
884 {
885 struct hist_trigger_data *test_data;
886 struct event_trigger_data *test;
887 struct hist_field *hist_field;
888
889 lockdep_assert_held(&event_mutex);
890
891 hist_field = find_var_field(hist_data, var_name);
892 if (hist_field)
893 return hist_field;
894
895 list_for_each_entry(test, &file->triggers, list) {
896 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
897 test_data = test->private_data;
898 hist_field = find_var_field(test_data, var_name);
899 if (hist_field)
900 return hist_field;
901 }
902 }
903
904 return NULL;
905 }
906
907 static struct trace_event_file *find_var_file(struct trace_array *tr,
908 char *system,
909 char *event_name,
910 char *var_name)
911 {
912 struct hist_trigger_data *var_hist_data;
913 struct hist_var_data *var_data;
914 struct trace_event_file *file, *found = NULL;
915
916 if (system)
917 return find_event_file(tr, system, event_name);
918
919 list_for_each_entry(var_data, &tr->hist_vars, list) {
920 var_hist_data = var_data->hist_data;
921 file = var_hist_data->event_file;
922 if (file == found)
923 continue;
924
925 if (find_var_field(var_hist_data, var_name)) {
926 if (found) {
927 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
928 return NULL;
929 }
930
931 found = file;
932 }
933 }
934
935 return found;
936 }
937
938 static struct hist_field *find_file_var(struct trace_event_file *file,
939 const char *var_name)
940 {
941 struct hist_trigger_data *test_data;
942 struct event_trigger_data *test;
943 struct hist_field *hist_field;
944
945 lockdep_assert_held(&event_mutex);
946
947 list_for_each_entry(test, &file->triggers, list) {
948 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
949 test_data = test->private_data;
950 hist_field = find_var_field(test_data, var_name);
951 if (hist_field)
952 return hist_field;
953 }
954 }
955
956 return NULL;
957 }
958
959 static struct hist_field *
960 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
961 {
962 struct trace_array *tr = hist_data->event_file->tr;
963 struct hist_field *hist_field, *found = NULL;
964 struct trace_event_file *file;
965 unsigned int i;
966
967 for (i = 0; i < hist_data->n_actions; i++) {
968 struct action_data *data = hist_data->actions[i];
969
970 if (data->handler == HANDLER_ONMATCH) {
971 char *system = data->match_data.event_system;
972 char *event_name = data->match_data.event;
973
974 file = find_var_file(tr, system, event_name, var_name);
975 if (!file)
976 continue;
977 hist_field = find_file_var(file, var_name);
978 if (hist_field) {
979 if (found) {
980 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
981 errpos(var_name));
982 return ERR_PTR(-EINVAL);
983 }
984
985 found = hist_field;
986 }
987 }
988 }
989 return found;
990 }
991
992 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
993 char *system,
994 char *event_name,
995 char *var_name)
996 {
997 struct trace_array *tr = hist_data->event_file->tr;
998 struct hist_field *hist_field = NULL;
999 struct trace_event_file *file;
1000
1001 if (!system || !event_name) {
1002 hist_field = find_match_var(hist_data, var_name);
1003 if (IS_ERR(hist_field))
1004 return NULL;
1005 if (hist_field)
1006 return hist_field;
1007 }
1008
1009 file = find_var_file(tr, system, event_name, var_name);
1010 if (!file)
1011 return NULL;
1012
1013 hist_field = find_file_var(file, var_name);
1014
1015 return hist_field;
1016 }
1017
1018 static u64 hist_field_var_ref(struct hist_field *hist_field,
1019 struct tracing_map_elt *elt,
1020 struct ring_buffer_event *rbe,
1021 void *event)
1022 {
1023 struct hist_elt_data *elt_data;
1024 u64 var_val = 0;
1025
1026 if (WARN_ON_ONCE(!elt))
1027 return var_val;
1028
1029 elt_data = elt->private_data;
1030 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1031
1032 return var_val;
1033 }
1034
1035 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1036 u64 *var_ref_vals, bool self)
1037 {
1038 struct hist_trigger_data *var_data;
1039 struct tracing_map_elt *var_elt;
1040 struct hist_field *hist_field;
1041 unsigned int i, var_idx;
1042 bool resolved = true;
1043 u64 var_val = 0;
1044
1045 for (i = 0; i < hist_data->n_var_refs; i++) {
1046 hist_field = hist_data->var_refs[i];
1047 var_idx = hist_field->var.idx;
1048 var_data = hist_field->var.hist_data;
1049
1050 if (var_data == NULL) {
1051 resolved = false;
1052 break;
1053 }
1054
1055 if ((self && var_data != hist_data) ||
1056 (!self && var_data == hist_data))
1057 continue;
1058
1059 var_elt = tracing_map_lookup(var_data->map, key);
1060 if (!var_elt) {
1061 resolved = false;
1062 break;
1063 }
1064
1065 if (!tracing_map_var_set(var_elt, var_idx)) {
1066 resolved = false;
1067 break;
1068 }
1069
1070 if (self || !hist_field->read_once)
1071 var_val = tracing_map_read_var(var_elt, var_idx);
1072 else
1073 var_val = tracing_map_read_var_once(var_elt, var_idx);
1074
1075 var_ref_vals[i] = var_val;
1076 }
1077
1078 return resolved;
1079 }
1080
1081 static const char *hist_field_name(struct hist_field *field,
1082 unsigned int level)
1083 {
1084 const char *field_name = "";
1085
1086 if (level > 1)
1087 return field_name;
1088
1089 if (field->field)
1090 field_name = field->field->name;
1091 else if (field->flags & HIST_FIELD_FL_LOG2 ||
1092 field->flags & HIST_FIELD_FL_ALIAS)
1093 field_name = hist_field_name(field->operands[0], ++level);
1094 else if (field->flags & HIST_FIELD_FL_CPU)
1095 field_name = "cpu";
1096 else if (field->flags & HIST_FIELD_FL_EXPR ||
1097 field->flags & HIST_FIELD_FL_VAR_REF) {
1098 if (field->system) {
1099 static char full_name[MAX_FILTER_STR_VAL];
1100
1101 strcat(full_name, field->system);
1102 strcat(full_name, ".");
1103 strcat(full_name, field->event_name);
1104 strcat(full_name, ".");
1105 strcat(full_name, field->name);
1106 field_name = full_name;
1107 } else
1108 field_name = field->name;
1109 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1110 field_name = "common_timestamp";
1111
1112 if (field_name == NULL)
1113 field_name = "";
1114
1115 return field_name;
1116 }
1117
1118 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1119 {
1120 hist_field_fn_t fn = NULL;
1121
1122 switch (field_size) {
1123 case 8:
1124 if (field_is_signed)
1125 fn = hist_field_s64;
1126 else
1127 fn = hist_field_u64;
1128 break;
1129 case 4:
1130 if (field_is_signed)
1131 fn = hist_field_s32;
1132 else
1133 fn = hist_field_u32;
1134 break;
1135 case 2:
1136 if (field_is_signed)
1137 fn = hist_field_s16;
1138 else
1139 fn = hist_field_u16;
1140 break;
1141 case 1:
1142 if (field_is_signed)
1143 fn = hist_field_s8;
1144 else
1145 fn = hist_field_u8;
1146 break;
1147 }
1148
1149 return fn;
1150 }
1151
1152 static int parse_map_size(char *str)
1153 {
1154 unsigned long size, map_bits;
1155 int ret;
1156
1157 ret = kstrtoul(str, 0, &size);
1158 if (ret)
1159 goto out;
1160
1161 map_bits = ilog2(roundup_pow_of_two(size));
1162 if (map_bits < TRACING_MAP_BITS_MIN ||
1163 map_bits > TRACING_MAP_BITS_MAX)
1164 ret = -EINVAL;
1165 else
1166 ret = map_bits;
1167 out:
1168 return ret;
1169 }
1170
1171 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1172 {
1173 unsigned int i;
1174
1175 if (!attrs)
1176 return;
1177
1178 for (i = 0; i < attrs->n_assignments; i++)
1179 kfree(attrs->assignment_str[i]);
1180
1181 for (i = 0; i < attrs->n_actions; i++)
1182 kfree(attrs->action_str[i]);
1183
1184 kfree(attrs->name);
1185 kfree(attrs->sort_key_str);
1186 kfree(attrs->keys_str);
1187 kfree(attrs->vals_str);
1188 kfree(attrs->clock);
1189 kfree(attrs);
1190 }
1191
1192 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1193 {
1194 int ret = -EINVAL;
1195
1196 if (attrs->n_actions >= HIST_ACTIONS_MAX)
1197 return ret;
1198
1199 if ((str_has_prefix(str, "onmatch(")) ||
1200 (str_has_prefix(str, "onmax(")) ||
1201 (str_has_prefix(str, "onchange("))) {
1202 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1203 if (!attrs->action_str[attrs->n_actions]) {
1204 ret = -ENOMEM;
1205 return ret;
1206 }
1207 attrs->n_actions++;
1208 ret = 0;
1209 }
1210 return ret;
1211 }
1212
1213 static int parse_assignment(struct trace_array *tr,
1214 char *str, struct hist_trigger_attrs *attrs)
1215 {
1216 int len, ret = 0;
1217
1218 if ((len = str_has_prefix(str, "key=")) ||
1219 (len = str_has_prefix(str, "keys="))) {
1220 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1221 if (!attrs->keys_str) {
1222 ret = -ENOMEM;
1223 goto out;
1224 }
1225 } else if ((len = str_has_prefix(str, "val=")) ||
1226 (len = str_has_prefix(str, "vals=")) ||
1227 (len = str_has_prefix(str, "values="))) {
1228 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1229 if (!attrs->vals_str) {
1230 ret = -ENOMEM;
1231 goto out;
1232 }
1233 } else if ((len = str_has_prefix(str, "sort="))) {
1234 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1235 if (!attrs->sort_key_str) {
1236 ret = -ENOMEM;
1237 goto out;
1238 }
1239 } else if (str_has_prefix(str, "name=")) {
1240 attrs->name = kstrdup(str, GFP_KERNEL);
1241 if (!attrs->name) {
1242 ret = -ENOMEM;
1243 goto out;
1244 }
1245 } else if ((len = str_has_prefix(str, "clock="))) {
1246 str += len;
1247
1248 str = strstrip(str);
1249 attrs->clock = kstrdup(str, GFP_KERNEL);
1250 if (!attrs->clock) {
1251 ret = -ENOMEM;
1252 goto out;
1253 }
1254 } else if ((len = str_has_prefix(str, "size="))) {
1255 int map_bits = parse_map_size(str + len);
1256
1257 if (map_bits < 0) {
1258 ret = map_bits;
1259 goto out;
1260 }
1261 attrs->map_bits = map_bits;
1262 } else {
1263 char *assignment;
1264
1265 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1266 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1267 ret = -EINVAL;
1268 goto out;
1269 }
1270
1271 assignment = kstrdup(str, GFP_KERNEL);
1272 if (!assignment) {
1273 ret = -ENOMEM;
1274 goto out;
1275 }
1276
1277 attrs->assignment_str[attrs->n_assignments++] = assignment;
1278 }
1279 out:
1280 return ret;
1281 }
1282
1283 static struct hist_trigger_attrs *
1284 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1285 {
1286 struct hist_trigger_attrs *attrs;
1287 int ret = 0;
1288
1289 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1290 if (!attrs)
1291 return ERR_PTR(-ENOMEM);
1292
1293 while (trigger_str) {
1294 char *str = strsep(&trigger_str, ":");
1295 char *rhs;
1296
1297 rhs = strchr(str, '=');
1298 if (rhs) {
1299 if (!strlen(++rhs)) {
1300 ret = -EINVAL;
1301 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1302 goto free;
1303 }
1304 ret = parse_assignment(tr, str, attrs);
1305 if (ret)
1306 goto free;
1307 } else if (strcmp(str, "pause") == 0)
1308 attrs->pause = true;
1309 else if ((strcmp(str, "cont") == 0) ||
1310 (strcmp(str, "continue") == 0))
1311 attrs->cont = true;
1312 else if (strcmp(str, "clear") == 0)
1313 attrs->clear = true;
1314 else {
1315 ret = parse_action(str, attrs);
1316 if (ret)
1317 goto free;
1318 }
1319 }
1320
1321 if (!attrs->keys_str) {
1322 ret = -EINVAL;
1323 goto free;
1324 }
1325
1326 if (!attrs->clock) {
1327 attrs->clock = kstrdup("global", GFP_KERNEL);
1328 if (!attrs->clock) {
1329 ret = -ENOMEM;
1330 goto free;
1331 }
1332 }
1333
1334 return attrs;
1335 free:
1336 destroy_hist_trigger_attrs(attrs);
1337
1338 return ERR_PTR(ret);
1339 }
1340
1341 static inline void save_comm(char *comm, struct task_struct *task)
1342 {
1343 if (!task->pid) {
1344 strcpy(comm, "<idle>");
1345 return;
1346 }
1347
1348 if (WARN_ON_ONCE(task->pid < 0)) {
1349 strcpy(comm, "<XXX>");
1350 return;
1351 }
1352
1353 strncpy(comm, task->comm, TASK_COMM_LEN);
1354 }
1355
1356 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1357 {
1358 unsigned int i;
1359
1360 for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1361 kfree(elt_data->field_var_str[i]);
1362
1363 kfree(elt_data->comm);
1364 kfree(elt_data);
1365 }
1366
1367 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1368 {
1369 struct hist_elt_data *elt_data = elt->private_data;
1370
1371 hist_elt_data_free(elt_data);
1372 }
1373
1374 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1375 {
1376 struct hist_trigger_data *hist_data = elt->map->private_data;
1377 unsigned int size = TASK_COMM_LEN;
1378 struct hist_elt_data *elt_data;
1379 struct hist_field *key_field;
1380 unsigned int i, n_str;
1381
1382 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1383 if (!elt_data)
1384 return -ENOMEM;
1385
1386 for_each_hist_key_field(i, hist_data) {
1387 key_field = hist_data->fields[i];
1388
1389 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1390 elt_data->comm = kzalloc(size, GFP_KERNEL);
1391 if (!elt_data->comm) {
1392 kfree(elt_data);
1393 return -ENOMEM;
1394 }
1395 break;
1396 }
1397 }
1398
1399 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
1400
1401 size = STR_VAR_LEN_MAX;
1402
1403 for (i = 0; i < n_str; i++) {
1404 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1405 if (!elt_data->field_var_str[i]) {
1406 hist_elt_data_free(elt_data);
1407 return -ENOMEM;
1408 }
1409 }
1410
1411 elt->private_data = elt_data;
1412
1413 return 0;
1414 }
1415
1416 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1417 {
1418 struct hist_elt_data *elt_data = elt->private_data;
1419
1420 if (elt_data->comm)
1421 save_comm(elt_data->comm, current);
1422 }
1423
1424 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1425 .elt_alloc = hist_trigger_elt_data_alloc,
1426 .elt_free = hist_trigger_elt_data_free,
1427 .elt_init = hist_trigger_elt_data_init,
1428 };
1429
1430 static const char *get_hist_field_flags(struct hist_field *hist_field)
1431 {
1432 const char *flags_str = NULL;
1433
1434 if (hist_field->flags & HIST_FIELD_FL_HEX)
1435 flags_str = "hex";
1436 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1437 flags_str = "sym";
1438 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1439 flags_str = "sym-offset";
1440 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1441 flags_str = "execname";
1442 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1443 flags_str = "syscall";
1444 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1445 flags_str = "log2";
1446 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1447 flags_str = "usecs";
1448
1449 return flags_str;
1450 }
1451
1452 static void expr_field_str(struct hist_field *field, char *expr)
1453 {
1454 if (field->flags & HIST_FIELD_FL_VAR_REF)
1455 strcat(expr, "$");
1456
1457 strcat(expr, hist_field_name(field, 0));
1458
1459 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1460 const char *flags_str = get_hist_field_flags(field);
1461
1462 if (flags_str) {
1463 strcat(expr, ".");
1464 strcat(expr, flags_str);
1465 }
1466 }
1467 }
1468
1469 static char *expr_str(struct hist_field *field, unsigned int level)
1470 {
1471 char *expr;
1472
1473 if (level > 1)
1474 return NULL;
1475
1476 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1477 if (!expr)
1478 return NULL;
1479
1480 if (!field->operands[0]) {
1481 expr_field_str(field, expr);
1482 return expr;
1483 }
1484
1485 if (field->operator == FIELD_OP_UNARY_MINUS) {
1486 char *subexpr;
1487
1488 strcat(expr, "-(");
1489 subexpr = expr_str(field->operands[0], ++level);
1490 if (!subexpr) {
1491 kfree(expr);
1492 return NULL;
1493 }
1494 strcat(expr, subexpr);
1495 strcat(expr, ")");
1496
1497 kfree(subexpr);
1498
1499 return expr;
1500 }
1501
1502 expr_field_str(field->operands[0], expr);
1503
1504 switch (field->operator) {
1505 case FIELD_OP_MINUS:
1506 strcat(expr, "-");
1507 break;
1508 case FIELD_OP_PLUS:
1509 strcat(expr, "+");
1510 break;
1511 default:
1512 kfree(expr);
1513 return NULL;
1514 }
1515
1516 expr_field_str(field->operands[1], expr);
1517
1518 return expr;
1519 }
1520
1521 static int contains_operator(char *str)
1522 {
1523 enum field_op_id field_op = FIELD_OP_NONE;
1524 char *op;
1525
1526 op = strpbrk(str, "+-");
1527 if (!op)
1528 return FIELD_OP_NONE;
1529
1530 switch (*op) {
1531 case '-':
1532 if (*str == '-')
1533 field_op = FIELD_OP_UNARY_MINUS;
1534 else
1535 field_op = FIELD_OP_MINUS;
1536 break;
1537 case '+':
1538 field_op = FIELD_OP_PLUS;
1539 break;
1540 default:
1541 break;
1542 }
1543
1544 return field_op;
1545 }
1546
1547 static void get_hist_field(struct hist_field *hist_field)
1548 {
1549 hist_field->ref++;
1550 }
1551
1552 static void __destroy_hist_field(struct hist_field *hist_field)
1553 {
1554 if (--hist_field->ref > 1)
1555 return;
1556
1557 kfree(hist_field->var.name);
1558 kfree(hist_field->name);
1559 kfree(hist_field->type);
1560
1561 kfree(hist_field->system);
1562 kfree(hist_field->event_name);
1563
1564 kfree(hist_field);
1565 }
1566
1567 static void destroy_hist_field(struct hist_field *hist_field,
1568 unsigned int level)
1569 {
1570 unsigned int i;
1571
1572 if (level > 3)
1573 return;
1574
1575 if (!hist_field)
1576 return;
1577
1578 if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1579 return; /* var refs will be destroyed separately */
1580
1581 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1582 destroy_hist_field(hist_field->operands[i], level + 1);
1583
1584 __destroy_hist_field(hist_field);
1585 }
1586
1587 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1588 struct ftrace_event_field *field,
1589 unsigned long flags,
1590 char *var_name)
1591 {
1592 struct hist_field *hist_field;
1593
1594 if (field && is_function_field(field))
1595 return NULL;
1596
1597 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1598 if (!hist_field)
1599 return NULL;
1600
1601 hist_field->ref = 1;
1602
1603 hist_field->hist_data = hist_data;
1604
1605 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1606 goto out; /* caller will populate */
1607
1608 if (flags & HIST_FIELD_FL_VAR_REF) {
1609 hist_field->fn = hist_field_var_ref;
1610 goto out;
1611 }
1612
1613 if (flags & HIST_FIELD_FL_HITCOUNT) {
1614 hist_field->fn = hist_field_counter;
1615 hist_field->size = sizeof(u64);
1616 hist_field->type = kstrdup("u64", GFP_KERNEL);
1617 if (!hist_field->type)
1618 goto free;
1619 goto out;
1620 }
1621
1622 if (flags & HIST_FIELD_FL_STACKTRACE) {
1623 hist_field->fn = hist_field_none;
1624 goto out;
1625 }
1626
1627 if (flags & HIST_FIELD_FL_LOG2) {
1628 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1629 hist_field->fn = hist_field_log2;
1630 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1631 hist_field->size = hist_field->operands[0]->size;
1632 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1633 if (!hist_field->type)
1634 goto free;
1635 goto out;
1636 }
1637
1638 if (flags & HIST_FIELD_FL_TIMESTAMP) {
1639 hist_field->fn = hist_field_timestamp;
1640 hist_field->size = sizeof(u64);
1641 hist_field->type = kstrdup("u64", GFP_KERNEL);
1642 if (!hist_field->type)
1643 goto free;
1644 goto out;
1645 }
1646
1647 if (flags & HIST_FIELD_FL_CPU) {
1648 hist_field->fn = hist_field_cpu;
1649 hist_field->size = sizeof(int);
1650 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1651 if (!hist_field->type)
1652 goto free;
1653 goto out;
1654 }
1655
1656 if (WARN_ON_ONCE(!field))
1657 goto out;
1658
1659 if (is_string_field(field)) {
1660 flags |= HIST_FIELD_FL_STRING;
1661
1662 hist_field->size = MAX_FILTER_STR_VAL;
1663 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1664 if (!hist_field->type)
1665 goto free;
1666
1667 if (field->filter_type == FILTER_STATIC_STRING)
1668 hist_field->fn = hist_field_string;
1669 else if (field->filter_type == FILTER_DYN_STRING)
1670 hist_field->fn = hist_field_dynstring;
1671 else
1672 hist_field->fn = hist_field_pstring;
1673 } else {
1674 hist_field->size = field->size;
1675 hist_field->is_signed = field->is_signed;
1676 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1677 if (!hist_field->type)
1678 goto free;
1679
1680 hist_field->fn = select_value_fn(field->size,
1681 field->is_signed);
1682 if (!hist_field->fn) {
1683 destroy_hist_field(hist_field, 0);
1684 return NULL;
1685 }
1686 }
1687 out:
1688 hist_field->field = field;
1689 hist_field->flags = flags;
1690
1691 if (var_name) {
1692 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1693 if (!hist_field->var.name)
1694 goto free;
1695 }
1696
1697 return hist_field;
1698 free:
1699 destroy_hist_field(hist_field, 0);
1700 return NULL;
1701 }
1702
1703 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1704 {
1705 unsigned int i;
1706
1707 for (i = 0; i < HIST_FIELDS_MAX; i++) {
1708 if (hist_data->fields[i]) {
1709 destroy_hist_field(hist_data->fields[i], 0);
1710 hist_data->fields[i] = NULL;
1711 }
1712 }
1713
1714 for (i = 0; i < hist_data->n_var_refs; i++) {
1715 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1716 __destroy_hist_field(hist_data->var_refs[i]);
1717 hist_data->var_refs[i] = NULL;
1718 }
1719 }
1720
1721 static int init_var_ref(struct hist_field *ref_field,
1722 struct hist_field *var_field,
1723 char *system, char *event_name)
1724 {
1725 int err = 0;
1726
1727 ref_field->var.idx = var_field->var.idx;
1728 ref_field->var.hist_data = var_field->hist_data;
1729 ref_field->size = var_field->size;
1730 ref_field->is_signed = var_field->is_signed;
1731 ref_field->flags |= var_field->flags &
1732 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1733
1734 if (system) {
1735 ref_field->system = kstrdup(system, GFP_KERNEL);
1736 if (!ref_field->system)
1737 return -ENOMEM;
1738 }
1739
1740 if (event_name) {
1741 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1742 if (!ref_field->event_name) {
1743 err = -ENOMEM;
1744 goto free;
1745 }
1746 }
1747
1748 if (var_field->var.name) {
1749 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1750 if (!ref_field->name) {
1751 err = -ENOMEM;
1752 goto free;
1753 }
1754 } else if (var_field->name) {
1755 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1756 if (!ref_field->name) {
1757 err = -ENOMEM;
1758 goto free;
1759 }
1760 }
1761
1762 ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1763 if (!ref_field->type) {
1764 err = -ENOMEM;
1765 goto free;
1766 }
1767 out:
1768 return err;
1769 free:
1770 kfree(ref_field->system);
1771 kfree(ref_field->event_name);
1772 kfree(ref_field->name);
1773
1774 goto out;
1775 }
1776
1777 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1778 struct hist_field *var_field)
1779 {
1780 struct hist_field *ref_field;
1781 int i;
1782
1783 for (i = 0; i < hist_data->n_var_refs; i++) {
1784 ref_field = hist_data->var_refs[i];
1785 if (ref_field->var.idx == var_field->var.idx &&
1786 ref_field->var.hist_data == var_field->hist_data)
1787 return i;
1788 }
1789
1790 return -ENOENT;
1791 }
1792
1793 /**
1794 * create_var_ref - Create a variable reference and attach it to trigger
1795 * @hist_data: The trigger that will be referencing the variable
1796 * @var_field: The VAR field to create a reference to
1797 * @system: The optional system string
1798 * @event_name: The optional event_name string
1799 *
1800 * Given a variable hist_field, create a VAR_REF hist_field that
1801 * represents a reference to it.
1802 *
1803 * This function also adds the reference to the trigger that
1804 * now references the variable.
1805 *
1806 * Return: The VAR_REF field if successful, NULL if not
1807 */
1808 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1809 struct hist_field *var_field,
1810 char *system, char *event_name)
1811 {
1812 unsigned long flags = HIST_FIELD_FL_VAR_REF;
1813 struct hist_field *ref_field;
1814 int i;
1815
1816 /* Check if the variable already exists */
1817 for (i = 0; i < hist_data->n_var_refs; i++) {
1818 ref_field = hist_data->var_refs[i];
1819 if (ref_field->var.idx == var_field->var.idx &&
1820 ref_field->var.hist_data == var_field->hist_data) {
1821 get_hist_field(ref_field);
1822 return ref_field;
1823 }
1824 }
1825
1826 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1827 if (ref_field) {
1828 if (init_var_ref(ref_field, var_field, system, event_name)) {
1829 destroy_hist_field(ref_field, 0);
1830 return NULL;
1831 }
1832
1833 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1834 ref_field->var_ref_idx = hist_data->n_var_refs++;
1835 }
1836
1837 return ref_field;
1838 }
1839
1840 static bool is_var_ref(char *var_name)
1841 {
1842 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1843 return false;
1844
1845 return true;
1846 }
1847
1848 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1849 char *var_name)
1850 {
1851 char *name, *field;
1852 unsigned int i;
1853
1854 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1855 name = hist_data->attrs->var_defs.name[i];
1856
1857 if (strcmp(var_name, name) == 0) {
1858 field = hist_data->attrs->var_defs.expr[i];
1859 if (contains_operator(field) || is_var_ref(field))
1860 continue;
1861 return field;
1862 }
1863 }
1864
1865 return NULL;
1866 }
1867
1868 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1869 char *system, char *event_name,
1870 char *var_name)
1871 {
1872 struct trace_event_call *call;
1873
1874 if (system && event_name) {
1875 call = hist_data->event_file->event_call;
1876
1877 if (strcmp(system, call->class->system) != 0)
1878 return NULL;
1879
1880 if (strcmp(event_name, trace_event_name(call)) != 0)
1881 return NULL;
1882 }
1883
1884 if (!!system != !!event_name)
1885 return NULL;
1886
1887 if (!is_var_ref(var_name))
1888 return NULL;
1889
1890 var_name++;
1891
1892 return field_name_from_var(hist_data, var_name);
1893 }
1894
1895 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1896 char *system, char *event_name,
1897 char *var_name)
1898 {
1899 struct hist_field *var_field = NULL, *ref_field = NULL;
1900 struct trace_array *tr = hist_data->event_file->tr;
1901
1902 if (!is_var_ref(var_name))
1903 return NULL;
1904
1905 var_name++;
1906
1907 var_field = find_event_var(hist_data, system, event_name, var_name);
1908 if (var_field)
1909 ref_field = create_var_ref(hist_data, var_field,
1910 system, event_name);
1911
1912 if (!ref_field)
1913 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1914
1915 return ref_field;
1916 }
1917
1918 static struct ftrace_event_field *
1919 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1920 char *field_str, unsigned long *flags)
1921 {
1922 struct ftrace_event_field *field = NULL;
1923 char *field_name, *modifier, *str;
1924 struct trace_array *tr = file->tr;
1925
1926 modifier = str = kstrdup(field_str, GFP_KERNEL);
1927 if (!modifier)
1928 return ERR_PTR(-ENOMEM);
1929
1930 field_name = strsep(&modifier, ".");
1931 if (modifier) {
1932 if (strcmp(modifier, "hex") == 0)
1933 *flags |= HIST_FIELD_FL_HEX;
1934 else if (strcmp(modifier, "sym") == 0)
1935 *flags |= HIST_FIELD_FL_SYM;
1936 else if (strcmp(modifier, "sym-offset") == 0)
1937 *flags |= HIST_FIELD_FL_SYM_OFFSET;
1938 else if ((strcmp(modifier, "execname") == 0) &&
1939 (strcmp(field_name, "common_pid") == 0))
1940 *flags |= HIST_FIELD_FL_EXECNAME;
1941 else if (strcmp(modifier, "syscall") == 0)
1942 *flags |= HIST_FIELD_FL_SYSCALL;
1943 else if (strcmp(modifier, "log2") == 0)
1944 *flags |= HIST_FIELD_FL_LOG2;
1945 else if (strcmp(modifier, "usecs") == 0)
1946 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1947 else {
1948 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1949 field = ERR_PTR(-EINVAL);
1950 goto out;
1951 }
1952 }
1953
1954 if (strcmp(field_name, "common_timestamp") == 0) {
1955 *flags |= HIST_FIELD_FL_TIMESTAMP;
1956 hist_data->enable_timestamps = true;
1957 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1958 hist_data->attrs->ts_in_usecs = true;
1959 } else if (strcmp(field_name, "cpu") == 0)
1960 *flags |= HIST_FIELD_FL_CPU;
1961 else {
1962 field = trace_find_event_field(file->event_call, field_name);
1963 if (!field || !field->size) {
1964 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1965 field = ERR_PTR(-EINVAL);
1966 goto out;
1967 }
1968 }
1969 out:
1970 kfree(str);
1971
1972 return field;
1973 }
1974
1975 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
1976 struct hist_field *var_ref,
1977 char *var_name)
1978 {
1979 struct hist_field *alias = NULL;
1980 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
1981
1982 alias = create_hist_field(hist_data, NULL, flags, var_name);
1983 if (!alias)
1984 return NULL;
1985
1986 alias->fn = var_ref->fn;
1987 alias->operands[0] = var_ref;
1988
1989 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
1990 destroy_hist_field(alias, 0);
1991 return NULL;
1992 }
1993
1994 alias->var_ref_idx = var_ref->var_ref_idx;
1995
1996 return alias;
1997 }
1998
1999 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2000 struct trace_event_file *file, char *str,
2001 unsigned long *flags, char *var_name)
2002 {
2003 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2004 struct ftrace_event_field *field = NULL;
2005 struct hist_field *hist_field = NULL;
2006 int ret = 0;
2007
2008 s = strchr(str, '.');
2009 if (s) {
2010 s = strchr(++s, '.');
2011 if (s) {
2012 ref_system = strsep(&str, ".");
2013 if (!str) {
2014 ret = -EINVAL;
2015 goto out;
2016 }
2017 ref_event = strsep(&str, ".");
2018 if (!str) {
2019 ret = -EINVAL;
2020 goto out;
2021 }
2022 ref_var = str;
2023 }
2024 }
2025
2026 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2027 if (!s) {
2028 hist_field = parse_var_ref(hist_data, ref_system,
2029 ref_event, ref_var);
2030 if (hist_field) {
2031 if (var_name) {
2032 hist_field = create_alias(hist_data, hist_field, var_name);
2033 if (!hist_field) {
2034 ret = -ENOMEM;
2035 goto out;
2036 }
2037 }
2038 return hist_field;
2039 }
2040 } else
2041 str = s;
2042
2043 field = parse_field(hist_data, file, str, flags);
2044 if (IS_ERR(field)) {
2045 ret = PTR_ERR(field);
2046 goto out;
2047 }
2048
2049 hist_field = create_hist_field(hist_data, field, *flags, var_name);
2050 if (!hist_field) {
2051 ret = -ENOMEM;
2052 goto out;
2053 }
2054
2055 return hist_field;
2056 out:
2057 return ERR_PTR(ret);
2058 }
2059
2060 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2061 struct trace_event_file *file,
2062 char *str, unsigned long flags,
2063 char *var_name, unsigned int level);
2064
2065 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2066 struct trace_event_file *file,
2067 char *str, unsigned long flags,
2068 char *var_name, unsigned int level)
2069 {
2070 struct hist_field *operand1, *expr = NULL;
2071 unsigned long operand_flags;
2072 int ret = 0;
2073 char *s;
2074
2075 /* we support only -(xxx) i.e. explicit parens required */
2076
2077 if (level > 3) {
2078 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2079 ret = -EINVAL;
2080 goto free;
2081 }
2082
2083 str++; /* skip leading '-' */
2084
2085 s = strchr(str, '(');
2086 if (s)
2087 str++;
2088 else {
2089 ret = -EINVAL;
2090 goto free;
2091 }
2092
2093 s = strrchr(str, ')');
2094 if (s)
2095 *s = '\0';
2096 else {
2097 ret = -EINVAL; /* no closing ')' */
2098 goto free;
2099 }
2100
2101 flags |= HIST_FIELD_FL_EXPR;
2102 expr = create_hist_field(hist_data, NULL, flags, var_name);
2103 if (!expr) {
2104 ret = -ENOMEM;
2105 goto free;
2106 }
2107
2108 operand_flags = 0;
2109 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2110 if (IS_ERR(operand1)) {
2111 ret = PTR_ERR(operand1);
2112 goto free;
2113 }
2114
2115 expr->flags |= operand1->flags &
2116 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2117 expr->fn = hist_field_unary_minus;
2118 expr->operands[0] = operand1;
2119 expr->operator = FIELD_OP_UNARY_MINUS;
2120 expr->name = expr_str(expr, 0);
2121 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2122 if (!expr->type) {
2123 ret = -ENOMEM;
2124 goto free;
2125 }
2126
2127 return expr;
2128 free:
2129 destroy_hist_field(expr, 0);
2130 return ERR_PTR(ret);
2131 }
2132
2133 static int check_expr_operands(struct trace_array *tr,
2134 struct hist_field *operand1,
2135 struct hist_field *operand2)
2136 {
2137 unsigned long operand1_flags = operand1->flags;
2138 unsigned long operand2_flags = operand2->flags;
2139
2140 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2141 (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2142 struct hist_field *var;
2143
2144 var = find_var_field(operand1->var.hist_data, operand1->name);
2145 if (!var)
2146 return -EINVAL;
2147 operand1_flags = var->flags;
2148 }
2149
2150 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2151 (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2152 struct hist_field *var;
2153
2154 var = find_var_field(operand2->var.hist_data, operand2->name);
2155 if (!var)
2156 return -EINVAL;
2157 operand2_flags = var->flags;
2158 }
2159
2160 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2161 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2162 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2163 return -EINVAL;
2164 }
2165
2166 return 0;
2167 }
2168
2169 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2170 struct trace_event_file *file,
2171 char *str, unsigned long flags,
2172 char *var_name, unsigned int level)
2173 {
2174 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2175 unsigned long operand_flags;
2176 int field_op, ret = -EINVAL;
2177 char *sep, *operand1_str;
2178
2179 if (level > 3) {
2180 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2181 return ERR_PTR(-EINVAL);
2182 }
2183
2184 field_op = contains_operator(str);
2185
2186 if (field_op == FIELD_OP_NONE)
2187 return parse_atom(hist_data, file, str, &flags, var_name);
2188
2189 if (field_op == FIELD_OP_UNARY_MINUS)
2190 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2191
2192 switch (field_op) {
2193 case FIELD_OP_MINUS:
2194 sep = "-";
2195 break;
2196 case FIELD_OP_PLUS:
2197 sep = "+";
2198 break;
2199 default:
2200 goto free;
2201 }
2202
2203 operand1_str = strsep(&str, sep);
2204 if (!operand1_str || !str)
2205 goto free;
2206
2207 operand_flags = 0;
2208 operand1 = parse_atom(hist_data, file, operand1_str,
2209 &operand_flags, NULL);
2210 if (IS_ERR(operand1)) {
2211 ret = PTR_ERR(operand1);
2212 operand1 = NULL;
2213 goto free;
2214 }
2215
2216 /* rest of string could be another expression e.g. b+c in a+b+c */
2217 operand_flags = 0;
2218 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2219 if (IS_ERR(operand2)) {
2220 ret = PTR_ERR(operand2);
2221 operand2 = NULL;
2222 goto free;
2223 }
2224
2225 ret = check_expr_operands(file->tr, operand1, operand2);
2226 if (ret)
2227 goto free;
2228
2229 flags |= HIST_FIELD_FL_EXPR;
2230
2231 flags |= operand1->flags &
2232 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2233
2234 expr = create_hist_field(hist_data, NULL, flags, var_name);
2235 if (!expr) {
2236 ret = -ENOMEM;
2237 goto free;
2238 }
2239
2240 operand1->read_once = true;
2241 operand2->read_once = true;
2242
2243 expr->operands[0] = operand1;
2244 expr->operands[1] = operand2;
2245 expr->operator = field_op;
2246 expr->name = expr_str(expr, 0);
2247 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2248 if (!expr->type) {
2249 ret = -ENOMEM;
2250 goto free;
2251 }
2252
2253 switch (field_op) {
2254 case FIELD_OP_MINUS:
2255 expr->fn = hist_field_minus;
2256 break;
2257 case FIELD_OP_PLUS:
2258 expr->fn = hist_field_plus;
2259 break;
2260 default:
2261 ret = -EINVAL;
2262 goto free;
2263 }
2264
2265 return expr;
2266 free:
2267 destroy_hist_field(operand1, 0);
2268 destroy_hist_field(operand2, 0);
2269 destroy_hist_field(expr, 0);
2270
2271 return ERR_PTR(ret);
2272 }
2273
2274 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2275 struct trace_event_file *file)
2276 {
2277 struct event_trigger_data *test;
2278
2279 lockdep_assert_held(&event_mutex);
2280
2281 list_for_each_entry(test, &file->triggers, list) {
2282 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2283 if (test->private_data == hist_data)
2284 return test->filter_str;
2285 }
2286 }
2287
2288 return NULL;
2289 }
2290
2291 static struct event_command trigger_hist_cmd;
2292 static int event_hist_trigger_func(struct event_command *cmd_ops,
2293 struct trace_event_file *file,
2294 char *glob, char *cmd, char *param);
2295
2296 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2297 struct hist_trigger_data *hist_data,
2298 unsigned int n_keys)
2299 {
2300 struct hist_field *target_hist_field, *hist_field;
2301 unsigned int n, i, j;
2302
2303 if (hist_data->n_fields - hist_data->n_vals != n_keys)
2304 return false;
2305
2306 i = hist_data->n_vals;
2307 j = target_hist_data->n_vals;
2308
2309 for (n = 0; n < n_keys; n++) {
2310 hist_field = hist_data->fields[i + n];
2311 target_hist_field = target_hist_data->fields[j + n];
2312
2313 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2314 return false;
2315 if (hist_field->size != target_hist_field->size)
2316 return false;
2317 if (hist_field->is_signed != target_hist_field->is_signed)
2318 return false;
2319 }
2320
2321 return true;
2322 }
2323
2324 static struct hist_trigger_data *
2325 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2326 struct trace_event_file *file)
2327 {
2328 struct hist_trigger_data *hist_data;
2329 struct event_trigger_data *test;
2330 unsigned int n_keys;
2331
2332 lockdep_assert_held(&event_mutex);
2333
2334 n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2335
2336 list_for_each_entry(test, &file->triggers, list) {
2337 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2338 hist_data = test->private_data;
2339
2340 if (compatible_keys(target_hist_data, hist_data, n_keys))
2341 return hist_data;
2342 }
2343 }
2344
2345 return NULL;
2346 }
2347
2348 static struct trace_event_file *event_file(struct trace_array *tr,
2349 char *system, char *event_name)
2350 {
2351 struct trace_event_file *file;
2352
2353 file = __find_event_file(tr, system, event_name);
2354 if (!file)
2355 return ERR_PTR(-EINVAL);
2356
2357 return file;
2358 }
2359
2360 static struct hist_field *
2361 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2362 char *system, char *event_name, char *field_name)
2363 {
2364 struct hist_field *event_var;
2365 char *synthetic_name;
2366
2367 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2368 if (!synthetic_name)
2369 return ERR_PTR(-ENOMEM);
2370
2371 strcpy(synthetic_name, "synthetic_");
2372 strcat(synthetic_name, field_name);
2373
2374 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2375
2376 kfree(synthetic_name);
2377
2378 return event_var;
2379 }
2380
2381 /**
2382 * create_field_var_hist - Automatically create a histogram and var for a field
2383 * @target_hist_data: The target hist trigger
2384 * @subsys_name: Optional subsystem name
2385 * @event_name: Optional event name
2386 * @field_name: The name of the field (and the resulting variable)
2387 *
2388 * Hist trigger actions fetch data from variables, not directly from
2389 * events. However, for convenience, users are allowed to directly
2390 * specify an event field in an action, which will be automatically
2391 * converted into a variable on their behalf.
2392
2393 * If a user specifies a field on an event that isn't the event the
2394 * histogram currently being defined (the target event histogram), the
2395 * only way that can be accomplished is if a new hist trigger is
2396 * created and the field variable defined on that.
2397 *
2398 * This function creates a new histogram compatible with the target
2399 * event (meaning a histogram with the same key as the target
2400 * histogram), and creates a variable for the specified field, but
2401 * with 'synthetic_' prepended to the variable name in order to avoid
2402 * collision with normal field variables.
2403 *
2404 * Return: The variable created for the field.
2405 */
2406 static struct hist_field *
2407 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2408 char *subsys_name, char *event_name, char *field_name)
2409 {
2410 struct trace_array *tr = target_hist_data->event_file->tr;
2411 struct hist_field *event_var = ERR_PTR(-EINVAL);
2412 struct hist_trigger_data *hist_data;
2413 unsigned int i, n, first = true;
2414 struct field_var_hist *var_hist;
2415 struct trace_event_file *file;
2416 struct hist_field *key_field;
2417 char *saved_filter;
2418 char *cmd;
2419 int ret;
2420
2421 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2422 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2423 return ERR_PTR(-EINVAL);
2424 }
2425
2426 file = event_file(tr, subsys_name, event_name);
2427
2428 if (IS_ERR(file)) {
2429 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2430 ret = PTR_ERR(file);
2431 return ERR_PTR(ret);
2432 }
2433
2434 /*
2435 * Look for a histogram compatible with target. We'll use the
2436 * found histogram specification to create a new matching
2437 * histogram with our variable on it. target_hist_data is not
2438 * yet a registered histogram so we can't use that.
2439 */
2440 hist_data = find_compatible_hist(target_hist_data, file);
2441 if (!hist_data) {
2442 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2443 return ERR_PTR(-EINVAL);
2444 }
2445
2446 /* See if a synthetic field variable has already been created */
2447 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2448 event_name, field_name);
2449 if (!IS_ERR_OR_NULL(event_var))
2450 return event_var;
2451
2452 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2453 if (!var_hist)
2454 return ERR_PTR(-ENOMEM);
2455
2456 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2457 if (!cmd) {
2458 kfree(var_hist);
2459 return ERR_PTR(-ENOMEM);
2460 }
2461
2462 /* Use the same keys as the compatible histogram */
2463 strcat(cmd, "keys=");
2464
2465 for_each_hist_key_field(i, hist_data) {
2466 key_field = hist_data->fields[i];
2467 if (!first)
2468 strcat(cmd, ",");
2469 strcat(cmd, key_field->field->name);
2470 first = false;
2471 }
2472
2473 /* Create the synthetic field variable specification */
2474 strcat(cmd, ":synthetic_");
2475 strcat(cmd, field_name);
2476 strcat(cmd, "=");
2477 strcat(cmd, field_name);
2478
2479 /* Use the same filter as the compatible histogram */
2480 saved_filter = find_trigger_filter(hist_data, file);
2481 if (saved_filter) {
2482 strcat(cmd, " if ");
2483 strcat(cmd, saved_filter);
2484 }
2485
2486 var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2487 if (!var_hist->cmd) {
2488 kfree(cmd);
2489 kfree(var_hist);
2490 return ERR_PTR(-ENOMEM);
2491 }
2492
2493 /* Save the compatible histogram information */
2494 var_hist->hist_data = hist_data;
2495
2496 /* Create the new histogram with our variable */
2497 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2498 "", "hist", cmd);
2499 if (ret) {
2500 kfree(cmd);
2501 kfree(var_hist->cmd);
2502 kfree(var_hist);
2503 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2504 return ERR_PTR(ret);
2505 }
2506
2507 kfree(cmd);
2508
2509 /* If we can't find the variable, something went wrong */
2510 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2511 event_name, field_name);
2512 if (IS_ERR_OR_NULL(event_var)) {
2513 kfree(var_hist->cmd);
2514 kfree(var_hist);
2515 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2516 return ERR_PTR(-EINVAL);
2517 }
2518
2519 n = target_hist_data->n_field_var_hists;
2520 target_hist_data->field_var_hists[n] = var_hist;
2521 target_hist_data->n_field_var_hists++;
2522
2523 return event_var;
2524 }
2525
2526 static struct hist_field *
2527 find_target_event_var(struct hist_trigger_data *hist_data,
2528 char *subsys_name, char *event_name, char *var_name)
2529 {
2530 struct trace_event_file *file = hist_data->event_file;
2531 struct hist_field *hist_field = NULL;
2532
2533 if (subsys_name) {
2534 struct trace_event_call *call;
2535
2536 if (!event_name)
2537 return NULL;
2538
2539 call = file->event_call;
2540
2541 if (strcmp(subsys_name, call->class->system) != 0)
2542 return NULL;
2543
2544 if (strcmp(event_name, trace_event_name(call)) != 0)
2545 return NULL;
2546 }
2547
2548 hist_field = find_var_field(hist_data, var_name);
2549
2550 return hist_field;
2551 }
2552
2553 static inline void __update_field_vars(struct tracing_map_elt *elt,
2554 struct ring_buffer_event *rbe,
2555 void *rec,
2556 struct field_var **field_vars,
2557 unsigned int n_field_vars,
2558 unsigned int field_var_str_start)
2559 {
2560 struct hist_elt_data *elt_data = elt->private_data;
2561 unsigned int i, j, var_idx;
2562 u64 var_val;
2563
2564 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2565 struct field_var *field_var = field_vars[i];
2566 struct hist_field *var = field_var->var;
2567 struct hist_field *val = field_var->val;
2568
2569 var_val = val->fn(val, elt, rbe, rec);
2570 var_idx = var->var.idx;
2571
2572 if (val->flags & HIST_FIELD_FL_STRING) {
2573 char *str = elt_data->field_var_str[j++];
2574 char *val_str = (char *)(uintptr_t)var_val;
2575
2576 strscpy(str, val_str, STR_VAR_LEN_MAX);
2577 var_val = (u64)(uintptr_t)str;
2578 }
2579 tracing_map_set_var(elt, var_idx, var_val);
2580 }
2581 }
2582
2583 static void update_field_vars(struct hist_trigger_data *hist_data,
2584 struct tracing_map_elt *elt,
2585 struct ring_buffer_event *rbe,
2586 void *rec)
2587 {
2588 __update_field_vars(elt, rbe, rec, hist_data->field_vars,
2589 hist_data->n_field_vars, 0);
2590 }
2591
2592 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2593 struct tracing_map_elt *elt, void *rec,
2594 struct ring_buffer_event *rbe, void *key,
2595 struct action_data *data, u64 *var_ref_vals)
2596 {
2597 __update_field_vars(elt, rbe, rec, hist_data->save_vars,
2598 hist_data->n_save_vars, hist_data->n_field_var_str);
2599 }
2600
2601 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2602 struct trace_event_file *file,
2603 char *name, int size, const char *type)
2604 {
2605 struct hist_field *var;
2606 int idx;
2607
2608 if (find_var(hist_data, file, name) && !hist_data->remove) {
2609 var = ERR_PTR(-EINVAL);
2610 goto out;
2611 }
2612
2613 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2614 if (!var) {
2615 var = ERR_PTR(-ENOMEM);
2616 goto out;
2617 }
2618
2619 idx = tracing_map_add_var(hist_data->map);
2620 if (idx < 0) {
2621 kfree(var);
2622 var = ERR_PTR(-EINVAL);
2623 goto out;
2624 }
2625
2626 var->ref = 1;
2627 var->flags = HIST_FIELD_FL_VAR;
2628 var->var.idx = idx;
2629 var->var.hist_data = var->hist_data = hist_data;
2630 var->size = size;
2631 var->var.name = kstrdup(name, GFP_KERNEL);
2632 var->type = kstrdup(type, GFP_KERNEL);
2633 if (!var->var.name || !var->type) {
2634 kfree(var->var.name);
2635 kfree(var->type);
2636 kfree(var);
2637 var = ERR_PTR(-ENOMEM);
2638 }
2639 out:
2640 return var;
2641 }
2642
2643 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2644 struct trace_event_file *file,
2645 char *field_name)
2646 {
2647 struct hist_field *val = NULL, *var = NULL;
2648 unsigned long flags = HIST_FIELD_FL_VAR;
2649 struct trace_array *tr = file->tr;
2650 struct field_var *field_var;
2651 int ret = 0;
2652
2653 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2654 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2655 ret = -EINVAL;
2656 goto err;
2657 }
2658
2659 val = parse_atom(hist_data, file, field_name, &flags, NULL);
2660 if (IS_ERR(val)) {
2661 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2662 ret = PTR_ERR(val);
2663 goto err;
2664 }
2665
2666 var = create_var(hist_data, file, field_name, val->size, val->type);
2667 if (IS_ERR(var)) {
2668 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2669 kfree(val);
2670 ret = PTR_ERR(var);
2671 goto err;
2672 }
2673
2674 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2675 if (!field_var) {
2676 kfree(val);
2677 kfree(var);
2678 ret = -ENOMEM;
2679 goto err;
2680 }
2681
2682 field_var->var = var;
2683 field_var->val = val;
2684 out:
2685 return field_var;
2686 err:
2687 field_var = ERR_PTR(ret);
2688 goto out;
2689 }
2690
2691 /**
2692 * create_target_field_var - Automatically create a variable for a field
2693 * @target_hist_data: The target hist trigger
2694 * @subsys_name: Optional subsystem name
2695 * @event_name: Optional event name
2696 * @var_name: The name of the field (and the resulting variable)
2697 *
2698 * Hist trigger actions fetch data from variables, not directly from
2699 * events. However, for convenience, users are allowed to directly
2700 * specify an event field in an action, which will be automatically
2701 * converted into a variable on their behalf.
2702
2703 * This function creates a field variable with the name var_name on
2704 * the hist trigger currently being defined on the target event. If
2705 * subsys_name and event_name are specified, this function simply
2706 * verifies that they do in fact match the target event subsystem and
2707 * event name.
2708 *
2709 * Return: The variable created for the field.
2710 */
2711 static struct field_var *
2712 create_target_field_var(struct hist_trigger_data *target_hist_data,
2713 char *subsys_name, char *event_name, char *var_name)
2714 {
2715 struct trace_event_file *file = target_hist_data->event_file;
2716
2717 if (subsys_name) {
2718 struct trace_event_call *call;
2719
2720 if (!event_name)
2721 return NULL;
2722
2723 call = file->event_call;
2724
2725 if (strcmp(subsys_name, call->class->system) != 0)
2726 return NULL;
2727
2728 if (strcmp(event_name, trace_event_name(call)) != 0)
2729 return NULL;
2730 }
2731
2732 return create_field_var(target_hist_data, file, var_name);
2733 }
2734
2735 static bool check_track_val_max(u64 track_val, u64 var_val)
2736 {
2737 if (var_val <= track_val)
2738 return false;
2739
2740 return true;
2741 }
2742
2743 static bool check_track_val_changed(u64 track_val, u64 var_val)
2744 {
2745 if (var_val == track_val)
2746 return false;
2747
2748 return true;
2749 }
2750
2751 static u64 get_track_val(struct hist_trigger_data *hist_data,
2752 struct tracing_map_elt *elt,
2753 struct action_data *data)
2754 {
2755 unsigned int track_var_idx = data->track_data.track_var->var.idx;
2756 u64 track_val;
2757
2758 track_val = tracing_map_read_var(elt, track_var_idx);
2759
2760 return track_val;
2761 }
2762
2763 static void save_track_val(struct hist_trigger_data *hist_data,
2764 struct tracing_map_elt *elt,
2765 struct action_data *data, u64 var_val)
2766 {
2767 unsigned int track_var_idx = data->track_data.track_var->var.idx;
2768
2769 tracing_map_set_var(elt, track_var_idx, var_val);
2770 }
2771
2772 static void save_track_data(struct hist_trigger_data *hist_data,
2773 struct tracing_map_elt *elt, void *rec,
2774 struct ring_buffer_event *rbe, void *key,
2775 struct action_data *data, u64 *var_ref_vals)
2776 {
2777 if (data->track_data.save_data)
2778 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2779 }
2780
2781 static bool check_track_val(struct tracing_map_elt *elt,
2782 struct action_data *data,
2783 u64 var_val)
2784 {
2785 struct hist_trigger_data *hist_data;
2786 u64 track_val;
2787
2788 hist_data = data->track_data.track_var->hist_data;
2789 track_val = get_track_val(hist_data, elt, data);
2790
2791 return data->track_data.check_val(track_val, var_val);
2792 }
2793
2794 #ifdef CONFIG_TRACER_SNAPSHOT
2795 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2796 {
2797 /* called with tr->max_lock held */
2798 struct track_data *track_data = tr->cond_snapshot->cond_data;
2799 struct hist_elt_data *elt_data, *track_elt_data;
2800 struct snapshot_context *context = cond_data;
2801 struct action_data *action;
2802 u64 track_val;
2803
2804 if (!track_data)
2805 return false;
2806
2807 action = track_data->action_data;
2808
2809 track_val = get_track_val(track_data->hist_data, context->elt,
2810 track_data->action_data);
2811
2812 if (!action->track_data.check_val(track_data->track_val, track_val))
2813 return false;
2814
2815 track_data->track_val = track_val;
2816 memcpy(track_data->key, context->key, track_data->key_len);
2817
2818 elt_data = context->elt->private_data;
2819 track_elt_data = track_data->elt.private_data;
2820 if (elt_data->comm)
2821 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2822
2823 track_data->updated = true;
2824
2825 return true;
2826 }
2827
2828 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2829 struct tracing_map_elt *elt, void *rec,
2830 struct ring_buffer_event *rbe, void *key,
2831 struct action_data *data,
2832 u64 *var_ref_vals)
2833 {
2834 struct trace_event_file *file = hist_data->event_file;
2835 struct snapshot_context context;
2836
2837 context.elt = elt;
2838 context.key = key;
2839
2840 tracing_snapshot_cond(file->tr, &context);
2841 }
2842
2843 static void hist_trigger_print_key(struct seq_file *m,
2844 struct hist_trigger_data *hist_data,
2845 void *key,
2846 struct tracing_map_elt *elt);
2847
2848 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2849 {
2850 unsigned int i;
2851
2852 if (!hist_data->n_actions)
2853 return NULL;
2854
2855 for (i = 0; i < hist_data->n_actions; i++) {
2856 struct action_data *data = hist_data->actions[i];
2857
2858 if (data->action == ACTION_SNAPSHOT)
2859 return data;
2860 }
2861
2862 return NULL;
2863 }
2864
2865 static void track_data_snapshot_print(struct seq_file *m,
2866 struct hist_trigger_data *hist_data)
2867 {
2868 struct trace_event_file *file = hist_data->event_file;
2869 struct track_data *track_data;
2870 struct action_data *action;
2871
2872 track_data = tracing_cond_snapshot_data(file->tr);
2873 if (!track_data)
2874 return;
2875
2876 if (!track_data->updated)
2877 return;
2878
2879 action = snapshot_action(hist_data);
2880 if (!action)
2881 return;
2882
2883 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n");
2884 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2885 action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2886 action->track_data.var_str, track_data->track_val);
2887
2888 seq_puts(m, "\ttriggered by event with key: ");
2889 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2890 seq_putc(m, '\n');
2891 }
2892 #else
2893 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2894 {
2895 return false;
2896 }
2897 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2898 struct tracing_map_elt *elt, void *rec,
2899 struct ring_buffer_event *rbe, void *key,
2900 struct action_data *data,
2901 u64 *var_ref_vals) {}
2902 static void track_data_snapshot_print(struct seq_file *m,
2903 struct hist_trigger_data *hist_data) {}
2904 #endif /* CONFIG_TRACER_SNAPSHOT */
2905
2906 static void track_data_print(struct seq_file *m,
2907 struct hist_trigger_data *hist_data,
2908 struct tracing_map_elt *elt,
2909 struct action_data *data)
2910 {
2911 u64 track_val = get_track_val(hist_data, elt, data);
2912 unsigned int i, save_var_idx;
2913
2914 if (data->handler == HANDLER_ONMAX)
2915 seq_printf(m, "\n\tmax: %10llu", track_val);
2916 else if (data->handler == HANDLER_ONCHANGE)
2917 seq_printf(m, "\n\tchanged: %10llu", track_val);
2918
2919 if (data->action == ACTION_SNAPSHOT)
2920 return;
2921
2922 for (i = 0; i < hist_data->n_save_vars; i++) {
2923 struct hist_field *save_val = hist_data->save_vars[i]->val;
2924 struct hist_field *save_var = hist_data->save_vars[i]->var;
2925 u64 val;
2926
2927 save_var_idx = save_var->var.idx;
2928
2929 val = tracing_map_read_var(elt, save_var_idx);
2930
2931 if (save_val->flags & HIST_FIELD_FL_STRING) {
2932 seq_printf(m, " %s: %-32s", save_var->var.name,
2933 (char *)(uintptr_t)(val));
2934 } else
2935 seq_printf(m, " %s: %10llu", save_var->var.name, val);
2936 }
2937 }
2938
2939 static void ontrack_action(struct hist_trigger_data *hist_data,
2940 struct tracing_map_elt *elt, void *rec,
2941 struct ring_buffer_event *rbe, void *key,
2942 struct action_data *data, u64 *var_ref_vals)
2943 {
2944 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2945
2946 if (check_track_val(elt, data, var_val)) {
2947 save_track_val(hist_data, elt, data, var_val);
2948 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2949 }
2950 }
2951
2952 static void action_data_destroy(struct action_data *data)
2953 {
2954 unsigned int i;
2955
2956 lockdep_assert_held(&event_mutex);
2957
2958 kfree(data->action_name);
2959
2960 for (i = 0; i < data->n_params; i++)
2961 kfree(data->params[i]);
2962
2963 if (data->synth_event)
2964 data->synth_event->ref--;
2965
2966 kfree(data->synth_event_name);
2967
2968 kfree(data);
2969 }
2970
2971 static void track_data_destroy(struct hist_trigger_data *hist_data,
2972 struct action_data *data)
2973 {
2974 struct trace_event_file *file = hist_data->event_file;
2975
2976 destroy_hist_field(data->track_data.track_var, 0);
2977
2978 if (data->action == ACTION_SNAPSHOT) {
2979 struct track_data *track_data;
2980
2981 track_data = tracing_cond_snapshot_data(file->tr);
2982 if (track_data && track_data->hist_data == hist_data) {
2983 tracing_snapshot_cond_disable(file->tr);
2984 track_data_free(track_data);
2985 }
2986 }
2987
2988 kfree(data->track_data.var_str);
2989
2990 action_data_destroy(data);
2991 }
2992
2993 static int action_create(struct hist_trigger_data *hist_data,
2994 struct action_data *data);
2995
2996 static int track_data_create(struct hist_trigger_data *hist_data,
2997 struct action_data *data)
2998 {
2999 struct hist_field *var_field, *ref_field, *track_var = NULL;
3000 struct trace_event_file *file = hist_data->event_file;
3001 struct trace_array *tr = file->tr;
3002 char *track_data_var_str;
3003 int ret = 0;
3004
3005 track_data_var_str = data->track_data.var_str;
3006 if (track_data_var_str[0] != '$') {
3007 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3008 return -EINVAL;
3009 }
3010 track_data_var_str++;
3011
3012 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3013 if (!var_field) {
3014 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3015 return -EINVAL;
3016 }
3017
3018 ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3019 if (!ref_field)
3020 return -ENOMEM;
3021
3022 data->track_data.var_ref = ref_field;
3023
3024 if (data->handler == HANDLER_ONMAX)
3025 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3026 if (IS_ERR(track_var)) {
3027 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3028 ret = PTR_ERR(track_var);
3029 goto out;
3030 }
3031
3032 if (data->handler == HANDLER_ONCHANGE)
3033 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3034 if (IS_ERR(track_var)) {
3035 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3036 ret = PTR_ERR(track_var);
3037 goto out;
3038 }
3039 data->track_data.track_var = track_var;
3040
3041 ret = action_create(hist_data, data);
3042 out:
3043 return ret;
3044 }
3045
3046 static int parse_action_params(struct trace_array *tr, char *params,
3047 struct action_data *data)
3048 {
3049 char *param, *saved_param;
3050 bool first_param = true;
3051 int ret = 0;
3052
3053 while (params) {
3054 if (data->n_params >= SYNTH_FIELDS_MAX) {
3055 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3056 goto out;
3057 }
3058
3059 param = strsep(&params, ",");
3060 if (!param) {
3061 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3062 ret = -EINVAL;
3063 goto out;
3064 }
3065
3066 param = strstrip(param);
3067 if (strlen(param) < 2) {
3068 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3069 ret = -EINVAL;
3070 goto out;
3071 }
3072
3073 saved_param = kstrdup(param, GFP_KERNEL);
3074 if (!saved_param) {
3075 ret = -ENOMEM;
3076 goto out;
3077 }
3078
3079 if (first_param && data->use_trace_keyword) {
3080 data->synth_event_name = saved_param;
3081 first_param = false;
3082 continue;
3083 }
3084 first_param = false;
3085
3086 data->params[data->n_params++] = saved_param;
3087 }
3088 out:
3089 return ret;
3090 }
3091
3092 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3093 enum handler_id handler)
3094 {
3095 char *action_name;
3096 int ret = 0;
3097
3098 strsep(&str, ".");
3099 if (!str) {
3100 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3101 ret = -EINVAL;
3102 goto out;
3103 }
3104
3105 action_name = strsep(&str, "(");
3106 if (!action_name || !str) {
3107 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3108 ret = -EINVAL;
3109 goto out;
3110 }
3111
3112 if (str_has_prefix(action_name, "save")) {
3113 char *params = strsep(&str, ")");
3114
3115 if (!params) {
3116 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3117 ret = -EINVAL;
3118 goto out;
3119 }
3120
3121 ret = parse_action_params(tr, params, data);
3122 if (ret)
3123 goto out;
3124
3125 if (handler == HANDLER_ONMAX)
3126 data->track_data.check_val = check_track_val_max;
3127 else if (handler == HANDLER_ONCHANGE)
3128 data->track_data.check_val = check_track_val_changed;
3129 else {
3130 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3131 ret = -EINVAL;
3132 goto out;
3133 }
3134
3135 data->track_data.save_data = save_track_data_vars;
3136 data->fn = ontrack_action;
3137 data->action = ACTION_SAVE;
3138 } else if (str_has_prefix(action_name, "snapshot")) {
3139 char *params = strsep(&str, ")");
3140
3141 if (!str) {
3142 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3143 ret = -EINVAL;
3144 goto out;
3145 }
3146
3147 if (handler == HANDLER_ONMAX)
3148 data->track_data.check_val = check_track_val_max;
3149 else if (handler == HANDLER_ONCHANGE)
3150 data->track_data.check_val = check_track_val_changed;
3151 else {
3152 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3153 ret = -EINVAL;
3154 goto out;
3155 }
3156
3157 data->track_data.save_data = save_track_data_snapshot;
3158 data->fn = ontrack_action;
3159 data->action = ACTION_SNAPSHOT;
3160 } else {
3161 char *params = strsep(&str, ")");
3162
3163 if (str_has_prefix(action_name, "trace"))
3164 data->use_trace_keyword = true;
3165
3166 if (params) {
3167 ret = parse_action_params(tr, params, data);
3168 if (ret)
3169 goto out;
3170 }
3171
3172 if (handler == HANDLER_ONMAX)
3173 data->track_data.check_val = check_track_val_max;
3174 else if (handler == HANDLER_ONCHANGE)
3175 data->track_data.check_val = check_track_val_changed;
3176
3177 if (handler != HANDLER_ONMATCH) {
3178 data->track_data.save_data = action_trace;
3179 data->fn = ontrack_action;
3180 } else
3181 data->fn = action_trace;
3182
3183 data->action = ACTION_TRACE;
3184 }
3185
3186 data->action_name = kstrdup(action_name, GFP_KERNEL);
3187 if (!data->action_name) {
3188 ret = -ENOMEM;
3189 goto out;
3190 }
3191
3192 data->handler = handler;
3193 out:
3194 return ret;
3195 }
3196
3197 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3198 char *str, enum handler_id handler)
3199 {
3200 struct action_data *data;
3201 int ret = -EINVAL;
3202 char *var_str;
3203
3204 data = kzalloc(sizeof(*data), GFP_KERNEL);
3205 if (!data)
3206 return ERR_PTR(-ENOMEM);
3207
3208 var_str = strsep(&str, ")");
3209 if (!var_str || !str) {
3210 ret = -EINVAL;
3211 goto free;
3212 }
3213
3214 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3215 if (!data->track_data.var_str) {
3216 ret = -ENOMEM;
3217 goto free;
3218 }
3219
3220 ret = action_parse(hist_data->event_file->tr, str, data, handler);
3221 if (ret)
3222 goto free;
3223 out:
3224 return data;
3225 free:
3226 track_data_destroy(hist_data, data);
3227 data = ERR_PTR(ret);
3228 goto out;
3229 }
3230
3231 static void onmatch_destroy(struct action_data *data)
3232 {
3233 kfree(data->match_data.event);
3234 kfree(data->match_data.event_system);
3235
3236 action_data_destroy(data);
3237 }
3238
3239 static void destroy_field_var(struct field_var *field_var)
3240 {
3241 if (!field_var)
3242 return;
3243
3244 destroy_hist_field(field_var->var, 0);
3245 destroy_hist_field(field_var->val, 0);
3246
3247 kfree(field_var);
3248 }
3249
3250 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3251 {
3252 unsigned int i;
3253
3254 for (i = 0; i < hist_data->n_field_vars; i++)
3255 destroy_field_var(hist_data->field_vars[i]);
3256
3257 for (i = 0; i < hist_data->n_save_vars; i++)
3258 destroy_field_var(hist_data->save_vars[i]);
3259 }
3260
3261 static void save_field_var(struct hist_trigger_data *hist_data,
3262 struct field_var *field_var)
3263 {
3264 hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3265
3266 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3267 hist_data->n_field_var_str++;
3268 }
3269
3270
3271 static int check_synth_field(struct synth_event *event,
3272 struct hist_field *hist_field,
3273 unsigned int field_pos)
3274 {
3275 struct synth_field *field;
3276
3277 if (field_pos >= event->n_fields)
3278 return -EINVAL;
3279
3280 field = event->fields[field_pos];
3281
3282 if (strcmp(field->type, hist_field->type) != 0) {
3283 if (field->size != hist_field->size ||
3284 field->is_signed != hist_field->is_signed)
3285 return -EINVAL;
3286 }
3287
3288 return 0;
3289 }
3290
3291 static struct hist_field *
3292 trace_action_find_var(struct hist_trigger_data *hist_data,
3293 struct action_data *data,
3294 char *system, char *event, char *var)
3295 {
3296 struct trace_array *tr = hist_data->event_file->tr;
3297 struct hist_field *hist_field;
3298
3299 var++; /* skip '$' */
3300
3301 hist_field = find_target_event_var(hist_data, system, event, var);
3302 if (!hist_field) {
3303 if (!system && data->handler == HANDLER_ONMATCH) {
3304 system = data->match_data.event_system;
3305 event = data->match_data.event;
3306 }
3307
3308 hist_field = find_event_var(hist_data, system, event, var);
3309 }
3310
3311 if (!hist_field)
3312 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3313
3314 return hist_field;
3315 }
3316
3317 static struct hist_field *
3318 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3319 struct action_data *data, char *system,
3320 char *event, char *var)
3321 {
3322 struct hist_field *hist_field = NULL;
3323 struct field_var *field_var;
3324
3325 /*
3326 * First try to create a field var on the target event (the
3327 * currently being defined). This will create a variable for
3328 * unqualified fields on the target event, or if qualified,
3329 * target fields that have qualified names matching the target.
3330 */
3331 field_var = create_target_field_var(hist_data, system, event, var);
3332
3333 if (field_var && !IS_ERR(field_var)) {
3334 save_field_var(hist_data, field_var);
3335 hist_field = field_var->var;
3336 } else {
3337 field_var = NULL;
3338 /*
3339 * If no explicit system.event is specfied, default to
3340 * looking for fields on the onmatch(system.event.xxx)
3341 * event.
3342 */
3343 if (!system && data->handler == HANDLER_ONMATCH) {
3344 system = data->match_data.event_system;
3345 event = data->match_data.event;
3346 }
3347
3348 /*
3349 * At this point, we're looking at a field on another
3350 * event. Because we can't modify a hist trigger on
3351 * another event to add a variable for a field, we need
3352 * to create a new trigger on that event and create the
3353 * variable at the same time.
3354 */
3355 hist_field = create_field_var_hist(hist_data, system, event, var);
3356 if (IS_ERR(hist_field))
3357 goto free;
3358 }
3359 out:
3360 return hist_field;
3361 free:
3362 destroy_field_var(field_var);
3363 hist_field = NULL;
3364 goto out;
3365 }
3366
3367 static int trace_action_create(struct hist_trigger_data *hist_data,
3368 struct action_data *data)
3369 {
3370 struct trace_array *tr = hist_data->event_file->tr;
3371 char *event_name, *param, *system = NULL;
3372 struct hist_field *hist_field, *var_ref;
3373 unsigned int i;
3374 unsigned int field_pos = 0;
3375 struct synth_event *event;
3376 char *synth_event_name;
3377 int var_ref_idx, ret = 0;
3378
3379 lockdep_assert_held(&event_mutex);
3380
3381 if (data->use_trace_keyword)
3382 synth_event_name = data->synth_event_name;
3383 else
3384 synth_event_name = data->action_name;
3385
3386 event = find_synth_event(synth_event_name);
3387 if (!event) {
3388 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3389 return -EINVAL;
3390 }
3391
3392 event->ref++;
3393
3394 for (i = 0; i < data->n_params; i++) {
3395 char *p;
3396
3397 p = param = kstrdup(data->params[i], GFP_KERNEL);
3398 if (!param) {
3399 ret = -ENOMEM;
3400 goto err;
3401 }
3402
3403 system = strsep(&param, ".");
3404 if (!param) {
3405 param = (char *)system;
3406 system = event_name = NULL;
3407 } else {
3408 event_name = strsep(&param, ".");
3409 if (!param) {
3410 kfree(p);
3411 ret = -EINVAL;
3412 goto err;
3413 }
3414 }
3415
3416 if (param[0] == '$')
3417 hist_field = trace_action_find_var(hist_data, data,
3418 system, event_name,
3419 param);
3420 else
3421 hist_field = trace_action_create_field_var(hist_data,
3422 data,
3423 system,
3424 event_name,
3425 param);
3426
3427 if (!hist_field) {
3428 kfree(p);
3429 ret = -EINVAL;
3430 goto err;
3431 }
3432
3433 if (check_synth_field(event, hist_field, field_pos) == 0) {
3434 var_ref = create_var_ref(hist_data, hist_field,
3435 system, event_name);
3436 if (!var_ref) {
3437 kfree(p);
3438 ret = -ENOMEM;
3439 goto err;
3440 }
3441
3442 var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3443 if (WARN_ON(var_ref_idx < 0)) {
3444 ret = var_ref_idx;
3445 goto err;
3446 }
3447
3448 data->var_ref_idx[i] = var_ref_idx;
3449
3450 field_pos++;
3451 kfree(p);
3452 continue;
3453 }
3454
3455 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3456 kfree(p);
3457 ret = -EINVAL;
3458 goto err;
3459 }
3460
3461 if (field_pos != event->n_fields) {
3462 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3463 ret = -EINVAL;
3464 goto err;
3465 }
3466
3467 data->synth_event = event;
3468 out:
3469 return ret;
3470 err:
3471 event->ref--;
3472
3473 goto out;
3474 }
3475
3476 static int action_create(struct hist_trigger_data *hist_data,
3477 struct action_data *data)
3478 {
3479 struct trace_event_file *file = hist_data->event_file;
3480 struct trace_array *tr = file->tr;
3481 struct track_data *track_data;
3482 struct field_var *field_var;
3483 unsigned int i;
3484 char *param;
3485 int ret = 0;
3486
3487 if (data->action == ACTION_TRACE)
3488 return trace_action_create(hist_data, data);
3489
3490 if (data->action == ACTION_SNAPSHOT) {
3491 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3492 if (IS_ERR(track_data)) {
3493 ret = PTR_ERR(track_data);
3494 goto out;
3495 }
3496
3497 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3498 cond_snapshot_update);
3499 if (ret)
3500 track_data_free(track_data);
3501
3502 goto out;
3503 }
3504
3505 if (data->action == ACTION_SAVE) {
3506 if (hist_data->n_save_vars) {
3507 ret = -EEXIST;
3508 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3509 goto out;
3510 }
3511
3512 for (i = 0; i < data->n_params; i++) {
3513 param = kstrdup(data->params[i], GFP_KERNEL);
3514 if (!param) {
3515 ret = -ENOMEM;
3516 goto out;
3517 }
3518
3519 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3520 if (IS_ERR(field_var)) {
3521 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3522 errpos(param));
3523 ret = PTR_ERR(field_var);
3524 kfree(param);
3525 goto out;
3526 }
3527
3528 hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3529 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3530 hist_data->n_save_var_str++;
3531 kfree(param);
3532 }
3533 }
3534 out:
3535 return ret;
3536 }
3537
3538 static int onmatch_create(struct hist_trigger_data *hist_data,
3539 struct action_data *data)
3540 {
3541 return action_create(hist_data, data);
3542 }
3543
3544 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3545 {
3546 char *match_event, *match_event_system;
3547 struct action_data *data;
3548 int ret = -EINVAL;
3549
3550 data = kzalloc(sizeof(*data), GFP_KERNEL);
3551 if (!data)
3552 return ERR_PTR(-ENOMEM);
3553
3554 match_event = strsep(&str, ")");
3555 if (!match_event || !str) {
3556 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3557 goto free;
3558 }
3559
3560 match_event_system = strsep(&match_event, ".");
3561 if (!match_event) {
3562 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3563 goto free;
3564 }
3565
3566 if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3567 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3568 goto free;
3569 }
3570
3571 data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3572 if (!data->match_data.event) {
3573 ret = -ENOMEM;
3574 goto free;
3575 }
3576
3577 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3578 if (!data->match_data.event_system) {
3579 ret = -ENOMEM;
3580 goto free;
3581 }
3582
3583 ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3584 if (ret)
3585 goto free;
3586 out:
3587 return data;
3588 free:
3589 onmatch_destroy(data);
3590 data = ERR_PTR(ret);
3591 goto out;
3592 }
3593
3594 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3595 {
3596 hist_data->fields[HITCOUNT_IDX] =
3597 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3598 if (!hist_data->fields[HITCOUNT_IDX])
3599 return -ENOMEM;
3600
3601 hist_data->n_vals++;
3602 hist_data->n_fields++;
3603
3604 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3605 return -EINVAL;
3606
3607 return 0;
3608 }
3609
3610 static int __create_val_field(struct hist_trigger_data *hist_data,
3611 unsigned int val_idx,
3612 struct trace_event_file *file,
3613 char *var_name, char *field_str,
3614 unsigned long flags)
3615 {
3616 struct hist_field *hist_field;
3617 int ret = 0;
3618
3619 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3620 if (IS_ERR(hist_field)) {
3621 ret = PTR_ERR(hist_field);
3622 goto out;
3623 }
3624
3625 hist_data->fields[val_idx] = hist_field;
3626
3627 ++hist_data->n_vals;
3628 ++hist_data->n_fields;
3629
3630 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3631 ret = -EINVAL;
3632 out:
3633 return ret;
3634 }
3635
3636 static int create_val_field(struct hist_trigger_data *hist_data,
3637 unsigned int val_idx,
3638 struct trace_event_file *file,
3639 char *field_str)
3640 {
3641 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3642 return -EINVAL;
3643
3644 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3645 }
3646
3647 static int create_var_field(struct hist_trigger_data *hist_data,
3648 unsigned int val_idx,
3649 struct trace_event_file *file,
3650 char *var_name, char *expr_str)
3651 {
3652 struct trace_array *tr = hist_data->event_file->tr;
3653 unsigned long flags = 0;
3654
3655 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3656 return -EINVAL;
3657
3658 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3659 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3660 return -EINVAL;
3661 }
3662
3663 flags |= HIST_FIELD_FL_VAR;
3664 hist_data->n_vars++;
3665 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3666 return -EINVAL;
3667
3668 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3669 }
3670
3671 static int create_val_fields(struct hist_trigger_data *hist_data,
3672 struct trace_event_file *file)
3673 {
3674 char *fields_str, *field_str;
3675 unsigned int i, j = 1;
3676 int ret;
3677
3678 ret = create_hitcount_val(hist_data);
3679 if (ret)
3680 goto out;
3681
3682 fields_str = hist_data->attrs->vals_str;
3683 if (!fields_str)
3684 goto out;
3685
3686 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3687 j < TRACING_MAP_VALS_MAX; i++) {
3688 field_str = strsep(&fields_str, ",");
3689 if (!field_str)
3690 break;
3691
3692 if (strcmp(field_str, "hitcount") == 0)
3693 continue;
3694
3695 ret = create_val_field(hist_data, j++, file, field_str);
3696 if (ret)
3697 goto out;
3698 }
3699
3700 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3701 ret = -EINVAL;
3702 out:
3703 return ret;
3704 }
3705
3706 static int create_key_field(struct hist_trigger_data *hist_data,
3707 unsigned int key_idx,
3708 unsigned int key_offset,
3709 struct trace_event_file *file,
3710 char *field_str)
3711 {
3712 struct trace_array *tr = hist_data->event_file->tr;
3713 struct hist_field *hist_field = NULL;
3714 unsigned long flags = 0;
3715 unsigned int key_size;
3716 int ret = 0;
3717
3718 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3719 return -EINVAL;
3720
3721 flags |= HIST_FIELD_FL_KEY;
3722
3723 if (strcmp(field_str, "stacktrace") == 0) {
3724 flags |= HIST_FIELD_FL_STACKTRACE;
3725 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3726 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3727 } else {
3728 hist_field = parse_expr(hist_data, file, field_str, flags,
3729 NULL, 0);
3730 if (IS_ERR(hist_field)) {
3731 ret = PTR_ERR(hist_field);
3732 goto out;
3733 }
3734
3735 if (field_has_hist_vars(hist_field, 0)) {
3736 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3737 destroy_hist_field(hist_field, 0);
3738 ret = -EINVAL;
3739 goto out;
3740 }
3741
3742 key_size = hist_field->size;
3743 }
3744
3745 hist_data->fields[key_idx] = hist_field;
3746
3747 key_size = ALIGN(key_size, sizeof(u64));
3748 hist_data->fields[key_idx]->size = key_size;
3749 hist_data->fields[key_idx]->offset = key_offset;
3750
3751 hist_data->key_size += key_size;
3752
3753 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3754 ret = -EINVAL;
3755 goto out;
3756 }
3757
3758 hist_data->n_keys++;
3759 hist_data->n_fields++;
3760
3761 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3762 return -EINVAL;
3763
3764 ret = key_size;
3765 out:
3766 return ret;
3767 }
3768
3769 static int create_key_fields(struct hist_trigger_data *hist_data,
3770 struct trace_event_file *file)
3771 {
3772 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3773 char *fields_str, *field_str;
3774 int ret = -EINVAL;
3775
3776 fields_str = hist_data->attrs->keys_str;
3777 if (!fields_str)
3778 goto out;
3779
3780 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3781 field_str = strsep(&fields_str, ",");
3782 if (!field_str)
3783 break;
3784 ret = create_key_field(hist_data, i, key_offset,
3785 file, field_str);
3786 if (ret < 0)
3787 goto out;
3788 key_offset += ret;
3789 }
3790 if (fields_str) {
3791 ret = -EINVAL;
3792 goto out;
3793 }
3794 ret = 0;
3795 out:
3796 return ret;
3797 }
3798
3799 static int create_var_fields(struct hist_trigger_data *hist_data,
3800 struct trace_event_file *file)
3801 {
3802 unsigned int i, j = hist_data->n_vals;
3803 int ret = 0;
3804
3805 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3806
3807 for (i = 0; i < n_vars; i++) {
3808 char *var_name = hist_data->attrs->var_defs.name[i];
3809 char *expr = hist_data->attrs->var_defs.expr[i];
3810
3811 ret = create_var_field(hist_data, j++, file, var_name, expr);
3812 if (ret)
3813 goto out;
3814 }
3815 out:
3816 return ret;
3817 }
3818
3819 static void free_var_defs(struct hist_trigger_data *hist_data)
3820 {
3821 unsigned int i;
3822
3823 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3824 kfree(hist_data->attrs->var_defs.name[i]);
3825 kfree(hist_data->attrs->var_defs.expr[i]);
3826 }
3827
3828 hist_data->attrs->var_defs.n_vars = 0;
3829 }
3830
3831 static int parse_var_defs(struct hist_trigger_data *hist_data)
3832 {
3833 struct trace_array *tr = hist_data->event_file->tr;
3834 char *s, *str, *var_name, *field_str;
3835 unsigned int i, j, n_vars = 0;
3836 int ret = 0;
3837
3838 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3839 str = hist_data->attrs->assignment_str[i];
3840 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3841 field_str = strsep(&str, ",");
3842 if (!field_str)
3843 break;
3844
3845 var_name = strsep(&field_str, "=");
3846 if (!var_name || !field_str) {
3847 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3848 errpos(var_name));
3849 ret = -EINVAL;
3850 goto free;
3851 }
3852
3853 if (n_vars == TRACING_MAP_VARS_MAX) {
3854 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3855 ret = -EINVAL;
3856 goto free;
3857 }
3858
3859 s = kstrdup(var_name, GFP_KERNEL);
3860 if (!s) {
3861 ret = -ENOMEM;
3862 goto free;
3863 }
3864 hist_data->attrs->var_defs.name[n_vars] = s;
3865
3866 s = kstrdup(field_str, GFP_KERNEL);
3867 if (!s) {
3868 kfree(hist_data->attrs->var_defs.name[n_vars]);
3869 ret = -ENOMEM;
3870 goto free;
3871 }
3872 hist_data->attrs->var_defs.expr[n_vars++] = s;
3873
3874 hist_data->attrs->var_defs.n_vars = n_vars;
3875 }
3876 }
3877
3878 return ret;
3879 free:
3880 free_var_defs(hist_data);
3881
3882 return ret;
3883 }
3884
3885 static int create_hist_fields(struct hist_trigger_data *hist_data,
3886 struct trace_event_file *file)
3887 {
3888 int ret;
3889
3890 ret = parse_var_defs(hist_data);
3891 if (ret)
3892 goto out;
3893
3894 ret = create_val_fields(hist_data, file);
3895 if (ret)
3896 goto out;
3897
3898 ret = create_var_fields(hist_data, file);
3899 if (ret)
3900 goto out;
3901
3902 ret = create_key_fields(hist_data, file);
3903 if (ret)
3904 goto out;
3905 out:
3906 free_var_defs(hist_data);
3907
3908 return ret;
3909 }
3910
3911 static int is_descending(struct trace_array *tr, const char *str)
3912 {
3913 if (!str)
3914 return 0;
3915
3916 if (strcmp(str, "descending") == 0)
3917 return 1;
3918
3919 if (strcmp(str, "ascending") == 0)
3920 return 0;
3921
3922 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3923
3924 return -EINVAL;
3925 }
3926
3927 static int create_sort_keys(struct hist_trigger_data *hist_data)
3928 {
3929 struct trace_array *tr = hist_data->event_file->tr;
3930 char *fields_str = hist_data->attrs->sort_key_str;
3931 struct tracing_map_sort_key *sort_key;
3932 int descending, ret = 0;
3933 unsigned int i, j, k;
3934
3935 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3936
3937 if (!fields_str)
3938 goto out;
3939
3940 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3941 struct hist_field *hist_field;
3942 char *field_str, *field_name;
3943 const char *test_name;
3944
3945 sort_key = &hist_data->sort_keys[i];
3946
3947 field_str = strsep(&fields_str, ",");
3948 if (!field_str)
3949 break;
3950
3951 if (!*field_str) {
3952 ret = -EINVAL;
3953 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3954 break;
3955 }
3956
3957 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
3958 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
3959 ret = -EINVAL;
3960 break;
3961 }
3962
3963 field_name = strsep(&field_str, ".");
3964 if (!field_name || !*field_name) {
3965 ret = -EINVAL;
3966 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3967 break;
3968 }
3969
3970 if (strcmp(field_name, "hitcount") == 0) {
3971 descending = is_descending(tr, field_str);
3972 if (descending < 0) {
3973 ret = descending;
3974 break;
3975 }
3976 sort_key->descending = descending;
3977 continue;
3978 }
3979
3980 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
3981 unsigned int idx;
3982
3983 hist_field = hist_data->fields[j];
3984 if (hist_field->flags & HIST_FIELD_FL_VAR)
3985 continue;
3986
3987 idx = k++;
3988
3989 test_name = hist_field_name(hist_field, 0);
3990
3991 if (strcmp(field_name, test_name) == 0) {
3992 sort_key->field_idx = idx;
3993 descending = is_descending(tr, field_str);
3994 if (descending < 0) {
3995 ret = descending;
3996 goto out;
3997 }
3998 sort_key->descending = descending;
3999 break;
4000 }
4001 }
4002 if (j == hist_data->n_fields) {
4003 ret = -EINVAL;
4004 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4005 break;
4006 }
4007 }
4008
4009 hist_data->n_sort_keys = i;
4010 out:
4011 return ret;
4012 }
4013
4014 static void destroy_actions(struct hist_trigger_data *hist_data)
4015 {
4016 unsigned int i;
4017
4018 for (i = 0; i < hist_data->n_actions; i++) {
4019 struct action_data *data = hist_data->actions[i];
4020
4021 if (data->handler == HANDLER_ONMATCH)
4022 onmatch_destroy(data);
4023 else if (data->handler == HANDLER_ONMAX ||
4024 data->handler == HANDLER_ONCHANGE)
4025 track_data_destroy(hist_data, data);
4026 else
4027 kfree(data);
4028 }
4029 }
4030
4031 static int parse_actions(struct hist_trigger_data *hist_data)
4032 {
4033 struct trace_array *tr = hist_data->event_file->tr;
4034 struct action_data *data;
4035 unsigned int i;
4036 int ret = 0;
4037 char *str;
4038 int len;
4039
4040 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4041 str = hist_data->attrs->action_str[i];
4042
4043 if ((len = str_has_prefix(str, "onmatch("))) {
4044 char *action_str = str + len;
4045
4046 data = onmatch_parse(tr, action_str);
4047 if (IS_ERR(data)) {
4048 ret = PTR_ERR(data);
4049 break;
4050 }
4051 } else if ((len = str_has_prefix(str, "onmax("))) {
4052 char *action_str = str + len;
4053
4054 data = track_data_parse(hist_data, action_str,
4055 HANDLER_ONMAX);
4056 if (IS_ERR(data)) {
4057 ret = PTR_ERR(data);
4058 break;
4059 }
4060 } else if ((len = str_has_prefix(str, "onchange("))) {
4061 char *action_str = str + len;
4062
4063 data = track_data_parse(hist_data, action_str,
4064 HANDLER_ONCHANGE);
4065 if (IS_ERR(data)) {
4066 ret = PTR_ERR(data);
4067 break;
4068 }
4069 } else {
4070 ret = -EINVAL;
4071 break;
4072 }
4073
4074 hist_data->actions[hist_data->n_actions++] = data;
4075 }
4076
4077 return ret;
4078 }
4079
4080 static int create_actions(struct hist_trigger_data *hist_data)
4081 {
4082 struct action_data *data;
4083 unsigned int i;
4084 int ret = 0;
4085
4086 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4087 data = hist_data->actions[i];
4088
4089 if (data->handler == HANDLER_ONMATCH) {
4090 ret = onmatch_create(hist_data, data);
4091 if (ret)
4092 break;
4093 } else if (data->handler == HANDLER_ONMAX ||
4094 data->handler == HANDLER_ONCHANGE) {
4095 ret = track_data_create(hist_data, data);
4096 if (ret)
4097 break;
4098 } else {
4099 ret = -EINVAL;
4100 break;
4101 }
4102 }
4103
4104 return ret;
4105 }
4106
4107 static void print_actions(struct seq_file *m,
4108 struct hist_trigger_data *hist_data,
4109 struct tracing_map_elt *elt)
4110 {
4111 unsigned int i;
4112
4113 for (i = 0; i < hist_data->n_actions; i++) {
4114 struct action_data *data = hist_data->actions[i];
4115
4116 if (data->action == ACTION_SNAPSHOT)
4117 continue;
4118
4119 if (data->handler == HANDLER_ONMAX ||
4120 data->handler == HANDLER_ONCHANGE)
4121 track_data_print(m, hist_data, elt, data);
4122 }
4123 }
4124
4125 static void print_action_spec(struct seq_file *m,
4126 struct hist_trigger_data *hist_data,
4127 struct action_data *data)
4128 {
4129 unsigned int i;
4130
4131 if (data->action == ACTION_SAVE) {
4132 for (i = 0; i < hist_data->n_save_vars; i++) {
4133 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4134 if (i < hist_data->n_save_vars - 1)
4135 seq_puts(m, ",");
4136 }
4137 } else if (data->action == ACTION_TRACE) {
4138 if (data->use_trace_keyword)
4139 seq_printf(m, "%s", data->synth_event_name);
4140 for (i = 0; i < data->n_params; i++) {
4141 if (i || data->use_trace_keyword)
4142 seq_puts(m, ",");
4143 seq_printf(m, "%s", data->params[i]);
4144 }
4145 }
4146 }
4147
4148 static void print_track_data_spec(struct seq_file *m,
4149 struct hist_trigger_data *hist_data,
4150 struct action_data *data)
4151 {
4152 if (data->handler == HANDLER_ONMAX)
4153 seq_puts(m, ":onmax(");
4154 else if (data->handler == HANDLER_ONCHANGE)
4155 seq_puts(m, ":onchange(");
4156 seq_printf(m, "%s", data->track_data.var_str);
4157 seq_printf(m, ").%s(", data->action_name);
4158
4159 print_action_spec(m, hist_data, data);
4160
4161 seq_puts(m, ")");
4162 }
4163
4164 static void print_onmatch_spec(struct seq_file *m,
4165 struct hist_trigger_data *hist_data,
4166 struct action_data *data)
4167 {
4168 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4169 data->match_data.event);
4170
4171 seq_printf(m, "%s(", data->action_name);
4172
4173 print_action_spec(m, hist_data, data);
4174
4175 seq_puts(m, ")");
4176 }
4177
4178 static bool actions_match(struct hist_trigger_data *hist_data,
4179 struct hist_trigger_data *hist_data_test)
4180 {
4181 unsigned int i, j;
4182
4183 if (hist_data->n_actions != hist_data_test->n_actions)
4184 return false;
4185
4186 for (i = 0; i < hist_data->n_actions; i++) {
4187 struct action_data *data = hist_data->actions[i];
4188 struct action_data *data_test = hist_data_test->actions[i];
4189 char *action_name, *action_name_test;
4190
4191 if (data->handler != data_test->handler)
4192 return false;
4193 if (data->action != data_test->action)
4194 return false;
4195
4196 if (data->n_params != data_test->n_params)
4197 return false;
4198
4199 for (j = 0; j < data->n_params; j++) {
4200 if (strcmp(data->params[j], data_test->params[j]) != 0)
4201 return false;
4202 }
4203
4204 if (data->use_trace_keyword)
4205 action_name = data->synth_event_name;
4206 else
4207 action_name = data->action_name;
4208
4209 if (data_test->use_trace_keyword)
4210 action_name_test = data_test->synth_event_name;
4211 else
4212 action_name_test = data_test->action_name;
4213
4214 if (strcmp(action_name, action_name_test) != 0)
4215 return false;
4216
4217 if (data->handler == HANDLER_ONMATCH) {
4218 if (strcmp(data->match_data.event_system,
4219 data_test->match_data.event_system) != 0)
4220 return false;
4221 if (strcmp(data->match_data.event,
4222 data_test->match_data.event) != 0)
4223 return false;
4224 } else if (data->handler == HANDLER_ONMAX ||
4225 data->handler == HANDLER_ONCHANGE) {
4226 if (strcmp(data->track_data.var_str,
4227 data_test->track_data.var_str) != 0)
4228 return false;
4229 }
4230 }
4231
4232 return true;
4233 }
4234
4235
4236 static void print_actions_spec(struct seq_file *m,
4237 struct hist_trigger_data *hist_data)
4238 {
4239 unsigned int i;
4240
4241 for (i = 0; i < hist_data->n_actions; i++) {
4242 struct action_data *data = hist_data->actions[i];
4243
4244 if (data->handler == HANDLER_ONMATCH)
4245 print_onmatch_spec(m, hist_data, data);
4246 else if (data->handler == HANDLER_ONMAX ||
4247 data->handler == HANDLER_ONCHANGE)
4248 print_track_data_spec(m, hist_data, data);
4249 }
4250 }
4251
4252 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4253 {
4254 unsigned int i;
4255
4256 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4257 kfree(hist_data->field_var_hists[i]->cmd);
4258 kfree(hist_data->field_var_hists[i]);
4259 }
4260 }
4261
4262 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4263 {
4264 if (!hist_data)
4265 return;
4266
4267 destroy_hist_trigger_attrs(hist_data->attrs);
4268 destroy_hist_fields(hist_data);
4269 tracing_map_destroy(hist_data->map);
4270
4271 destroy_actions(hist_data);
4272 destroy_field_vars(hist_data);
4273 destroy_field_var_hists(hist_data);
4274
4275 kfree(hist_data);
4276 }
4277
4278 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4279 {
4280 struct tracing_map *map = hist_data->map;
4281 struct ftrace_event_field *field;
4282 struct hist_field *hist_field;
4283 int i, idx = 0;
4284
4285 for_each_hist_field(i, hist_data) {
4286 hist_field = hist_data->fields[i];
4287 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4288 tracing_map_cmp_fn_t cmp_fn;
4289
4290 field = hist_field->field;
4291
4292 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4293 cmp_fn = tracing_map_cmp_none;
4294 else if (!field)
4295 cmp_fn = tracing_map_cmp_num(hist_field->size,
4296 hist_field->is_signed);
4297 else if (is_string_field(field))
4298 cmp_fn = tracing_map_cmp_string;
4299 else
4300 cmp_fn = tracing_map_cmp_num(field->size,
4301 field->is_signed);
4302 idx = tracing_map_add_key_field(map,
4303 hist_field->offset,
4304 cmp_fn);
4305 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4306 idx = tracing_map_add_sum_field(map);
4307
4308 if (idx < 0)
4309 return idx;
4310
4311 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4312 idx = tracing_map_add_var(map);
4313 if (idx < 0)
4314 return idx;
4315 hist_field->var.idx = idx;
4316 hist_field->var.hist_data = hist_data;
4317 }
4318 }
4319
4320 return 0;
4321 }
4322
4323 static struct hist_trigger_data *
4324 create_hist_data(unsigned int map_bits,
4325 struct hist_trigger_attrs *attrs,
4326 struct trace_event_file *file,
4327 bool remove)
4328 {
4329 const struct tracing_map_ops *map_ops = NULL;
4330 struct hist_trigger_data *hist_data;
4331 int ret = 0;
4332
4333 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4334 if (!hist_data)
4335 return ERR_PTR(-ENOMEM);
4336
4337 hist_data->attrs = attrs;
4338 hist_data->remove = remove;
4339 hist_data->event_file = file;
4340
4341 ret = parse_actions(hist_data);
4342 if (ret)
4343 goto free;
4344
4345 ret = create_hist_fields(hist_data, file);
4346 if (ret)
4347 goto free;
4348
4349 ret = create_sort_keys(hist_data);
4350 if (ret)
4351 goto free;
4352
4353 map_ops = &hist_trigger_elt_data_ops;
4354
4355 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4356 map_ops, hist_data);
4357 if (IS_ERR(hist_data->map)) {
4358 ret = PTR_ERR(hist_data->map);
4359 hist_data->map = NULL;
4360 goto free;
4361 }
4362
4363 ret = create_tracing_map_fields(hist_data);
4364 if (ret)
4365 goto free;
4366 out:
4367 return hist_data;
4368 free:
4369 hist_data->attrs = NULL;
4370
4371 destroy_hist_data(hist_data);
4372
4373 hist_data = ERR_PTR(ret);
4374
4375 goto out;
4376 }
4377
4378 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4379 struct tracing_map_elt *elt, void *rec,
4380 struct ring_buffer_event *rbe,
4381 u64 *var_ref_vals)
4382 {
4383 struct hist_elt_data *elt_data;
4384 struct hist_field *hist_field;
4385 unsigned int i, var_idx;
4386 u64 hist_val;
4387
4388 elt_data = elt->private_data;
4389 elt_data->var_ref_vals = var_ref_vals;
4390
4391 for_each_hist_val_field(i, hist_data) {
4392 hist_field = hist_data->fields[i];
4393 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4394 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4395 var_idx = hist_field->var.idx;
4396 tracing_map_set_var(elt, var_idx, hist_val);
4397 continue;
4398 }
4399 tracing_map_update_sum(elt, i, hist_val);
4400 }
4401
4402 for_each_hist_key_field(i, hist_data) {
4403 hist_field = hist_data->fields[i];
4404 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4405 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4406 var_idx = hist_field->var.idx;
4407 tracing_map_set_var(elt, var_idx, hist_val);
4408 }
4409 }
4410
4411 update_field_vars(hist_data, elt, rbe, rec);
4412 }
4413
4414 static inline void add_to_key(char *compound_key, void *key,
4415 struct hist_field *key_field, void *rec)
4416 {
4417 size_t size = key_field->size;
4418
4419 if (key_field->flags & HIST_FIELD_FL_STRING) {
4420 struct ftrace_event_field *field;
4421
4422 field = key_field->field;
4423 if (field->filter_type == FILTER_DYN_STRING)
4424 size = *(u32 *)(rec + field->offset) >> 16;
4425 else if (field->filter_type == FILTER_PTR_STRING)
4426 size = strlen(key);
4427 else if (field->filter_type == FILTER_STATIC_STRING)
4428 size = field->size;
4429
4430 /* ensure NULL-termination */
4431 if (size > key_field->size - 1)
4432 size = key_field->size - 1;
4433
4434 strncpy(compound_key + key_field->offset, (char *)key, size);
4435 } else
4436 memcpy(compound_key + key_field->offset, key, size);
4437 }
4438
4439 static void
4440 hist_trigger_actions(struct hist_trigger_data *hist_data,
4441 struct tracing_map_elt *elt, void *rec,
4442 struct ring_buffer_event *rbe, void *key,
4443 u64 *var_ref_vals)
4444 {
4445 struct action_data *data;
4446 unsigned int i;
4447
4448 for (i = 0; i < hist_data->n_actions; i++) {
4449 data = hist_data->actions[i];
4450 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4451 }
4452 }
4453
4454 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4455 struct ring_buffer_event *rbe)
4456 {
4457 struct hist_trigger_data *hist_data = data->private_data;
4458 bool use_compound_key = (hist_data->n_keys > 1);
4459 unsigned long entries[HIST_STACKTRACE_DEPTH];
4460 u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4461 char compound_key[HIST_KEY_SIZE_MAX];
4462 struct tracing_map_elt *elt = NULL;
4463 struct hist_field *key_field;
4464 u64 field_contents;
4465 void *key = NULL;
4466 unsigned int i;
4467
4468 memset(compound_key, 0, hist_data->key_size);
4469
4470 for_each_hist_key_field(i, hist_data) {
4471 key_field = hist_data->fields[i];
4472
4473 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4474 memset(entries, 0, HIST_STACKTRACE_SIZE);
4475 stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4476 HIST_STACKTRACE_SKIP);
4477 key = entries;
4478 } else {
4479 field_contents = key_field->fn(key_field, elt, rbe, rec);
4480 if (key_field->flags & HIST_FIELD_FL_STRING) {
4481 key = (void *)(unsigned long)field_contents;
4482 use_compound_key = true;
4483 } else
4484 key = (void *)&field_contents;
4485 }
4486
4487 if (use_compound_key)
4488 add_to_key(compound_key, key, key_field, rec);
4489 }
4490
4491 if (use_compound_key)
4492 key = compound_key;
4493
4494 if (hist_data->n_var_refs &&
4495 !resolve_var_refs(hist_data, key, var_ref_vals, false))
4496 return;
4497
4498 elt = tracing_map_insert(hist_data->map, key);
4499 if (!elt)
4500 return;
4501
4502 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4503
4504 if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4505 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
4506 }
4507
4508 static void hist_trigger_stacktrace_print(struct seq_file *m,
4509 unsigned long *stacktrace_entries,
4510 unsigned int max_entries)
4511 {
4512 char str[KSYM_SYMBOL_LEN];
4513 unsigned int spaces = 8;
4514 unsigned int i;
4515
4516 for (i = 0; i < max_entries; i++) {
4517 if (!stacktrace_entries[i])
4518 return;
4519
4520 seq_printf(m, "%*c", 1 + spaces, ' ');
4521 sprint_symbol(str, stacktrace_entries[i]);
4522 seq_printf(m, "%s\n", str);
4523 }
4524 }
4525
4526 static void hist_trigger_print_key(struct seq_file *m,
4527 struct hist_trigger_data *hist_data,
4528 void *key,
4529 struct tracing_map_elt *elt)
4530 {
4531 struct hist_field *key_field;
4532 char str[KSYM_SYMBOL_LEN];
4533 bool multiline = false;
4534 const char *field_name;
4535 unsigned int i;
4536 u64 uval;
4537
4538 seq_puts(m, "{ ");
4539
4540 for_each_hist_key_field(i, hist_data) {
4541 key_field = hist_data->fields[i];
4542
4543 if (i > hist_data->n_vals)
4544 seq_puts(m, ", ");
4545
4546 field_name = hist_field_name(key_field, 0);
4547
4548 if (key_field->flags & HIST_FIELD_FL_HEX) {
4549 uval = *(u64 *)(key + key_field->offset);
4550 seq_printf(m, "%s: %llx", field_name, uval);
4551 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4552 uval = *(u64 *)(key + key_field->offset);
4553 sprint_symbol_no_offset(str, uval);
4554 seq_printf(m, "%s: [%llx] %-45s", field_name,
4555 uval, str);
4556 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4557 uval = *(u64 *)(key + key_field->offset);
4558 sprint_symbol(str, uval);
4559 seq_printf(m, "%s: [%llx] %-55s", field_name,
4560 uval, str);
4561 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4562 struct hist_elt_data *elt_data = elt->private_data;
4563 char *comm;
4564
4565 if (WARN_ON_ONCE(!elt_data))
4566 return;
4567
4568 comm = elt_data->comm;
4569
4570 uval = *(u64 *)(key + key_field->offset);
4571 seq_printf(m, "%s: %-16s[%10llu]", field_name,
4572 comm, uval);
4573 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4574 const char *syscall_name;
4575
4576 uval = *(u64 *)(key + key_field->offset);
4577 syscall_name = get_syscall_name(uval);
4578 if (!syscall_name)
4579 syscall_name = "unknown_syscall";
4580
4581 seq_printf(m, "%s: %-30s[%3llu]", field_name,
4582 syscall_name, uval);
4583 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4584 seq_puts(m, "stacktrace:\n");
4585 hist_trigger_stacktrace_print(m,
4586 key + key_field->offset,
4587 HIST_STACKTRACE_DEPTH);
4588 multiline = true;
4589 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4590 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4591 *(u64 *)(key + key_field->offset));
4592 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4593 seq_printf(m, "%s: %-50s", field_name,
4594 (char *)(key + key_field->offset));
4595 } else {
4596 uval = *(u64 *)(key + key_field->offset);
4597 seq_printf(m, "%s: %10llu", field_name, uval);
4598 }
4599 }
4600
4601 if (!multiline)
4602 seq_puts(m, " ");
4603
4604 seq_puts(m, "}");
4605 }
4606
4607 static void hist_trigger_entry_print(struct seq_file *m,
4608 struct hist_trigger_data *hist_data,
4609 void *key,
4610 struct tracing_map_elt *elt)
4611 {
4612 const char *field_name;
4613 unsigned int i;
4614
4615 hist_trigger_print_key(m, hist_data, key, elt);
4616
4617 seq_printf(m, " hitcount: %10llu",
4618 tracing_map_read_sum(elt, HITCOUNT_IDX));
4619
4620 for (i = 1; i < hist_data->n_vals; i++) {
4621 field_name = hist_field_name(hist_data->fields[i], 0);
4622
4623 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4624 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4625 continue;
4626
4627 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4628 seq_printf(m, " %s: %10llx", field_name,
4629 tracing_map_read_sum(elt, i));
4630 } else {
4631 seq_printf(m, " %s: %10llu", field_name,
4632 tracing_map_read_sum(elt, i));
4633 }
4634 }
4635
4636 print_actions(m, hist_data, elt);
4637
4638 seq_puts(m, "\n");
4639 }
4640
4641 static int print_entries(struct seq_file *m,
4642 struct hist_trigger_data *hist_data)
4643 {
4644 struct tracing_map_sort_entry **sort_entries = NULL;
4645 struct tracing_map *map = hist_data->map;
4646 int i, n_entries;
4647
4648 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4649 hist_data->n_sort_keys,
4650 &sort_entries);
4651 if (n_entries < 0)
4652 return n_entries;
4653
4654 for (i = 0; i < n_entries; i++)
4655 hist_trigger_entry_print(m, hist_data,
4656 sort_entries[i]->key,
4657 sort_entries[i]->elt);
4658
4659 tracing_map_destroy_sort_entries(sort_entries, n_entries);
4660
4661 return n_entries;
4662 }
4663
4664 static void hist_trigger_show(struct seq_file *m,
4665 struct event_trigger_data *data, int n)
4666 {
4667 struct hist_trigger_data *hist_data;
4668 int n_entries;
4669
4670 if (n > 0)
4671 seq_puts(m, "\n\n");
4672
4673 seq_puts(m, "# event histogram\n#\n# trigger info: ");
4674 data->ops->print(m, data->ops, data);
4675 seq_puts(m, "#\n\n");
4676
4677 hist_data = data->private_data;
4678 n_entries = print_entries(m, hist_data);
4679 if (n_entries < 0)
4680 n_entries = 0;
4681
4682 track_data_snapshot_print(m, hist_data);
4683
4684 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
4685 (u64)atomic64_read(&hist_data->map->hits),
4686 n_entries, (u64)atomic64_read(&hist_data->map->drops));
4687 }
4688
4689 static int hist_show(struct seq_file *m, void *v)
4690 {
4691 struct event_trigger_data *data;
4692 struct trace_event_file *event_file;
4693 int n = 0, ret = 0;
4694
4695 mutex_lock(&event_mutex);
4696
4697 event_file = event_file_data(m->private);
4698 if (unlikely(!event_file)) {
4699 ret = -ENODEV;
4700 goto out_unlock;
4701 }
4702
4703 list_for_each_entry(data, &event_file->triggers, list) {
4704 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4705 hist_trigger_show(m, data, n++);
4706 }
4707
4708 out_unlock:
4709 mutex_unlock(&event_mutex);
4710
4711 return ret;
4712 }
4713
4714 static int event_hist_open(struct inode *inode, struct file *file)
4715 {
4716 int ret;
4717
4718 ret = security_locked_down(LOCKDOWN_TRACEFS);
4719 if (ret)
4720 return ret;
4721
4722 return single_open(file, hist_show, file);
4723 }
4724
4725 const struct file_operations event_hist_fops = {
4726 .open = event_hist_open,
4727 .read = seq_read,
4728 .llseek = seq_lseek,
4729 .release = single_release,
4730 };
4731
4732 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4733 static void hist_field_debug_show_flags(struct seq_file *m,
4734 unsigned long flags)
4735 {
4736 seq_puts(m, " flags:\n");
4737
4738 if (flags & HIST_FIELD_FL_KEY)
4739 seq_puts(m, " HIST_FIELD_FL_KEY\n");
4740 else if (flags & HIST_FIELD_FL_HITCOUNT)
4741 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n");
4742 else if (flags & HIST_FIELD_FL_VAR)
4743 seq_puts(m, " HIST_FIELD_FL_VAR\n");
4744 else if (flags & HIST_FIELD_FL_VAR_REF)
4745 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n");
4746 else
4747 seq_puts(m, " VAL: normal u64 value\n");
4748
4749 if (flags & HIST_FIELD_FL_ALIAS)
4750 seq_puts(m, " HIST_FIELD_FL_ALIAS\n");
4751 }
4752
4753 static int hist_field_debug_show(struct seq_file *m,
4754 struct hist_field *field, unsigned long flags)
4755 {
4756 if ((field->flags & flags) != flags) {
4757 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4758 return -EINVAL;
4759 }
4760
4761 hist_field_debug_show_flags(m, field->flags);
4762 if (field->field)
4763 seq_printf(m, " ftrace_event_field name: %s\n",
4764 field->field->name);
4765
4766 if (field->flags & HIST_FIELD_FL_VAR) {
4767 seq_printf(m, " var.name: %s\n", field->var.name);
4768 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
4769 field->var.idx);
4770 }
4771
4772 if (field->flags & HIST_FIELD_FL_ALIAS)
4773 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
4774 field->var_ref_idx);
4775
4776 if (field->flags & HIST_FIELD_FL_VAR_REF) {
4777 seq_printf(m, " name: %s\n", field->name);
4778 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
4779 field->var.idx);
4780 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data);
4781 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
4782 field->var_ref_idx);
4783 if (field->system)
4784 seq_printf(m, " system: %s\n", field->system);
4785 if (field->event_name)
4786 seq_printf(m, " event_name: %s\n", field->event_name);
4787 }
4788
4789 seq_printf(m, " type: %s\n", field->type);
4790 seq_printf(m, " size: %u\n", field->size);
4791 seq_printf(m, " is_signed: %u\n", field->is_signed);
4792
4793 return 0;
4794 }
4795
4796 static int field_var_debug_show(struct seq_file *m,
4797 struct field_var *field_var, unsigned int i,
4798 bool save_vars)
4799 {
4800 const char *vars_name = save_vars ? "save_vars" : "field_vars";
4801 struct hist_field *field;
4802 int ret = 0;
4803
4804 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i);
4805
4806 field = field_var->var;
4807
4808 seq_printf(m, "\n %s[%d].var:\n", vars_name, i);
4809
4810 hist_field_debug_show_flags(m, field->flags);
4811 seq_printf(m, " var.name: %s\n", field->var.name);
4812 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
4813 field->var.idx);
4814
4815 field = field_var->val;
4816
4817 seq_printf(m, "\n %s[%d].val:\n", vars_name, i);
4818 if (field->field)
4819 seq_printf(m, " ftrace_event_field name: %s\n",
4820 field->field->name);
4821 else {
4822 ret = -EINVAL;
4823 goto out;
4824 }
4825
4826 seq_printf(m, " type: %s\n", field->type);
4827 seq_printf(m, " size: %u\n", field->size);
4828 seq_printf(m, " is_signed: %u\n", field->is_signed);
4829 out:
4830 return ret;
4831 }
4832
4833 static int hist_action_debug_show(struct seq_file *m,
4834 struct action_data *data, int i)
4835 {
4836 int ret = 0;
4837
4838 if (data->handler == HANDLER_ONMAX ||
4839 data->handler == HANDLER_ONCHANGE) {
4840 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i);
4841 ret = hist_field_debug_show(m, data->track_data.var_ref,
4842 HIST_FIELD_FL_VAR_REF);
4843 if (ret)
4844 goto out;
4845
4846 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i);
4847 ret = hist_field_debug_show(m, data->track_data.track_var,
4848 HIST_FIELD_FL_VAR);
4849 if (ret)
4850 goto out;
4851 }
4852
4853 if (data->handler == HANDLER_ONMATCH) {
4854 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n",
4855 i, data->match_data.event_system);
4856 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n",
4857 i, data->match_data.event);
4858 }
4859 out:
4860 return ret;
4861 }
4862
4863 static int hist_actions_debug_show(struct seq_file *m,
4864 struct hist_trigger_data *hist_data)
4865 {
4866 int i, ret = 0;
4867
4868 if (hist_data->n_actions)
4869 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n");
4870
4871 for (i = 0; i < hist_data->n_actions; i++) {
4872 struct action_data *action = hist_data->actions[i];
4873
4874 ret = hist_action_debug_show(m, action, i);
4875 if (ret)
4876 goto out;
4877 }
4878
4879 if (hist_data->n_save_vars)
4880 seq_puts(m, "\n save action variables (save() params):\n");
4881
4882 for (i = 0; i < hist_data->n_save_vars; i++) {
4883 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4884 if (ret)
4885 goto out;
4886 }
4887 out:
4888 return ret;
4889 }
4890
4891 static void hist_trigger_debug_show(struct seq_file *m,
4892 struct event_trigger_data *data, int n)
4893 {
4894 struct hist_trigger_data *hist_data;
4895 int i, ret;
4896
4897 if (n > 0)
4898 seq_puts(m, "\n\n");
4899
4900 seq_puts(m, "# event histogram\n#\n# trigger info: ");
4901 data->ops->print(m, data->ops, data);
4902 seq_puts(m, "#\n\n");
4903
4904 hist_data = data->private_data;
4905
4906 seq_printf(m, "hist_data: %p\n\n", hist_data);
4907 seq_printf(m, " n_vals: %u\n", hist_data->n_vals);
4908 seq_printf(m, " n_keys: %u\n", hist_data->n_keys);
4909 seq_printf(m, " n_fields: %u\n", hist_data->n_fields);
4910
4911 seq_puts(m, "\n val fields:\n\n");
4912
4913 seq_puts(m, " hist_data->fields[0]:\n");
4914 ret = hist_field_debug_show(m, hist_data->fields[0],
4915 HIST_FIELD_FL_HITCOUNT);
4916 if (ret)
4917 return;
4918
4919 for (i = 1; i < hist_data->n_vals; i++) {
4920 seq_printf(m, "\n hist_data->fields[%d]:\n", i);
4921 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4922 if (ret)
4923 return;
4924 }
4925
4926 seq_puts(m, "\n key fields:\n");
4927
4928 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
4929 seq_printf(m, "\n hist_data->fields[%d]:\n", i);
4930 ret = hist_field_debug_show(m, hist_data->fields[i],
4931 HIST_FIELD_FL_KEY);
4932 if (ret)
4933 return;
4934 }
4935
4936 if (hist_data->n_var_refs)
4937 seq_puts(m, "\n variable reference fields:\n");
4938
4939 for (i = 0; i < hist_data->n_var_refs; i++) {
4940 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i);
4941 ret = hist_field_debug_show(m, hist_data->var_refs[i],
4942 HIST_FIELD_FL_VAR_REF);
4943 if (ret)
4944 return;
4945 }
4946
4947 if (hist_data->n_field_vars)
4948 seq_puts(m, "\n field variables:\n");
4949
4950 for (i = 0; i < hist_data->n_field_vars; i++) {
4951 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
4952 if (ret)
4953 return;
4954 }
4955
4956 ret = hist_actions_debug_show(m, hist_data);
4957 if (ret)
4958 return;
4959 }
4960
4961 static int hist_debug_show(struct seq_file *m, void *v)
4962 {
4963 struct event_trigger_data *data;
4964 struct trace_event_file *event_file;
4965 int n = 0, ret = 0;
4966
4967 mutex_lock(&event_mutex);
4968
4969 event_file = event_file_data(m->private);
4970 if (unlikely(!event_file)) {
4971 ret = -ENODEV;
4972 goto out_unlock;
4973 }
4974
4975 list_for_each_entry(data, &event_file->triggers, list) {
4976 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4977 hist_trigger_debug_show(m, data, n++);
4978 }
4979
4980 out_unlock:
4981 mutex_unlock(&event_mutex);
4982
4983 return ret;
4984 }
4985
4986 static int event_hist_debug_open(struct inode *inode, struct file *file)
4987 {
4988 int ret;
4989
4990 ret = security_locked_down(LOCKDOWN_TRACEFS);
4991 if (ret)
4992 return ret;
4993
4994 return single_open(file, hist_debug_show, file);
4995 }
4996
4997 const struct file_operations event_hist_debug_fops = {
4998 .open = event_hist_debug_open,
4999 .read = seq_read,
5000 .llseek = seq_lseek,
5001 .release = single_release,
5002 };
5003 #endif
5004
5005 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5006 {
5007 const char *field_name = hist_field_name(hist_field, 0);
5008
5009 if (hist_field->var.name)
5010 seq_printf(m, "%s=", hist_field->var.name);
5011
5012 if (hist_field->flags & HIST_FIELD_FL_CPU)
5013 seq_puts(m, "cpu");
5014 else if (field_name) {
5015 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5016 hist_field->flags & HIST_FIELD_FL_ALIAS)
5017 seq_putc(m, '$');
5018 seq_printf(m, "%s", field_name);
5019 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5020 seq_puts(m, "common_timestamp");
5021
5022 if (hist_field->flags) {
5023 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5024 !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5025 const char *flags = get_hist_field_flags(hist_field);
5026
5027 if (flags)
5028 seq_printf(m, ".%s", flags);
5029 }
5030 }
5031 }
5032
5033 static int event_hist_trigger_print(struct seq_file *m,
5034 struct event_trigger_ops *ops,
5035 struct event_trigger_data *data)
5036 {
5037 struct hist_trigger_data *hist_data = data->private_data;
5038 struct hist_field *field;
5039 bool have_var = false;
5040 unsigned int i;
5041
5042 seq_puts(m, "hist:");
5043
5044 if (data->name)
5045 seq_printf(m, "%s:", data->name);
5046
5047 seq_puts(m, "keys=");
5048
5049 for_each_hist_key_field(i, hist_data) {
5050 field = hist_data->fields[i];
5051
5052 if (i > hist_data->n_vals)
5053 seq_puts(m, ",");
5054
5055 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5056 seq_puts(m, "stacktrace");
5057 else
5058 hist_field_print(m, field);
5059 }
5060
5061 seq_puts(m, ":vals=");
5062
5063 for_each_hist_val_field(i, hist_data) {
5064 field = hist_data->fields[i];
5065 if (field->flags & HIST_FIELD_FL_VAR) {
5066 have_var = true;
5067 continue;
5068 }
5069
5070 if (i == HITCOUNT_IDX)
5071 seq_puts(m, "hitcount");
5072 else {
5073 seq_puts(m, ",");
5074 hist_field_print(m, field);
5075 }
5076 }
5077
5078 if (have_var) {
5079 unsigned int n = 0;
5080
5081 seq_puts(m, ":");
5082
5083 for_each_hist_val_field(i, hist_data) {
5084 field = hist_data->fields[i];
5085
5086 if (field->flags & HIST_FIELD_FL_VAR) {
5087 if (n++)
5088 seq_puts(m, ",");
5089 hist_field_print(m, field);
5090 }
5091 }
5092 }
5093
5094 seq_puts(m, ":sort=");
5095
5096 for (i = 0; i < hist_data->n_sort_keys; i++) {
5097 struct tracing_map_sort_key *sort_key;
5098 unsigned int idx, first_key_idx;
5099
5100 /* skip VAR vals */
5101 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5102
5103 sort_key = &hist_data->sort_keys[i];
5104 idx = sort_key->field_idx;
5105
5106 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5107 return -EINVAL;
5108
5109 if (i > 0)
5110 seq_puts(m, ",");
5111
5112 if (idx == HITCOUNT_IDX)
5113 seq_puts(m, "hitcount");
5114 else {
5115 if (idx >= first_key_idx)
5116 idx += hist_data->n_vars;
5117 hist_field_print(m, hist_data->fields[idx]);
5118 }
5119
5120 if (sort_key->descending)
5121 seq_puts(m, ".descending");
5122 }
5123 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5124 if (hist_data->enable_timestamps)
5125 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5126
5127 print_actions_spec(m, hist_data);
5128
5129 if (data->filter_str)
5130 seq_printf(m, " if %s", data->filter_str);
5131
5132 if (data->paused)
5133 seq_puts(m, " [paused]");
5134 else
5135 seq_puts(m, " [active]");
5136
5137 seq_putc(m, '\n');
5138
5139 return 0;
5140 }
5141
5142 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5143 struct event_trigger_data *data)
5144 {
5145 struct hist_trigger_data *hist_data = data->private_data;
5146
5147 if (!data->ref && hist_data->attrs->name)
5148 save_named_trigger(hist_data->attrs->name, data);
5149
5150 data->ref++;
5151
5152 return 0;
5153 }
5154
5155 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5156 {
5157 struct trace_event_file *file;
5158 unsigned int i;
5159 char *cmd;
5160 int ret;
5161
5162 for (i = 0; i < hist_data->n_field_var_hists; i++) {
5163 file = hist_data->field_var_hists[i]->hist_data->event_file;
5164 cmd = hist_data->field_var_hists[i]->cmd;
5165 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5166 "!hist", "hist", cmd);
5167 }
5168 }
5169
5170 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5171 struct event_trigger_data *data)
5172 {
5173 struct hist_trigger_data *hist_data = data->private_data;
5174
5175 if (WARN_ON_ONCE(data->ref <= 0))
5176 return;
5177
5178 data->ref--;
5179 if (!data->ref) {
5180 if (data->name)
5181 del_named_trigger(data);
5182
5183 trigger_data_free(data);
5184
5185 remove_hist_vars(hist_data);
5186
5187 unregister_field_var_hists(hist_data);
5188
5189 destroy_hist_data(hist_data);
5190 }
5191 }
5192
5193 static struct event_trigger_ops event_hist_trigger_ops = {
5194 .func = event_hist_trigger,
5195 .print = event_hist_trigger_print,
5196 .init = event_hist_trigger_init,
5197 .free = event_hist_trigger_free,
5198 };
5199
5200 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5201 struct event_trigger_data *data)
5202 {
5203 data->ref++;
5204
5205 save_named_trigger(data->named_data->name, data);
5206
5207 event_hist_trigger_init(ops, data->named_data);
5208
5209 return 0;
5210 }
5211
5212 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5213 struct event_trigger_data *data)
5214 {
5215 if (WARN_ON_ONCE(data->ref <= 0))
5216 return;
5217
5218 event_hist_trigger_free(ops, data->named_data);
5219
5220 data->ref--;
5221 if (!data->ref) {
5222 del_named_trigger(data);
5223 trigger_data_free(data);
5224 }
5225 }
5226
5227 static struct event_trigger_ops event_hist_trigger_named_ops = {
5228 .func = event_hist_trigger,
5229 .print = event_hist_trigger_print,
5230 .init = event_hist_trigger_named_init,
5231 .free = event_hist_trigger_named_free,
5232 };
5233
5234 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5235 char *param)
5236 {
5237 return &event_hist_trigger_ops;
5238 }
5239
5240 static void hist_clear(struct event_trigger_data *data)
5241 {
5242 struct hist_trigger_data *hist_data = data->private_data;
5243
5244 if (data->name)
5245 pause_named_trigger(data);
5246
5247 tracepoint_synchronize_unregister();
5248
5249 tracing_map_clear(hist_data->map);
5250
5251 if (data->name)
5252 unpause_named_trigger(data);
5253 }
5254
5255 static bool compatible_field(struct ftrace_event_field *field,
5256 struct ftrace_event_field *test_field)
5257 {
5258 if (field == test_field)
5259 return true;
5260 if (field == NULL || test_field == NULL)
5261 return false;
5262 if (strcmp(field->name, test_field->name) != 0)
5263 return false;
5264 if (strcmp(field->type, test_field->type) != 0)
5265 return false;
5266 if (field->size != test_field->size)
5267 return false;
5268 if (field->is_signed != test_field->is_signed)
5269 return false;
5270
5271 return true;
5272 }
5273
5274 static bool hist_trigger_match(struct event_trigger_data *data,
5275 struct event_trigger_data *data_test,
5276 struct event_trigger_data *named_data,
5277 bool ignore_filter)
5278 {
5279 struct tracing_map_sort_key *sort_key, *sort_key_test;
5280 struct hist_trigger_data *hist_data, *hist_data_test;
5281 struct hist_field *key_field, *key_field_test;
5282 unsigned int i;
5283
5284 if (named_data && (named_data != data_test) &&
5285 (named_data != data_test->named_data))
5286 return false;
5287
5288 if (!named_data && is_named_trigger(data_test))
5289 return false;
5290
5291 hist_data = data->private_data;
5292 hist_data_test = data_test->private_data;
5293
5294 if (hist_data->n_vals != hist_data_test->n_vals ||
5295 hist_data->n_fields != hist_data_test->n_fields ||
5296 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5297 return false;
5298
5299 if (!ignore_filter) {
5300 if ((data->filter_str && !data_test->filter_str) ||
5301 (!data->filter_str && data_test->filter_str))
5302 return false;
5303 }
5304
5305 for_each_hist_field(i, hist_data) {
5306 key_field = hist_data->fields[i];
5307 key_field_test = hist_data_test->fields[i];
5308
5309 if (key_field->flags != key_field_test->flags)
5310 return false;
5311 if (!compatible_field(key_field->field, key_field_test->field))
5312 return false;
5313 if (key_field->offset != key_field_test->offset)
5314 return false;
5315 if (key_field->size != key_field_test->size)
5316 return false;
5317 if (key_field->is_signed != key_field_test->is_signed)
5318 return false;
5319 if (!!key_field->var.name != !!key_field_test->var.name)
5320 return false;
5321 if (key_field->var.name &&
5322 strcmp(key_field->var.name, key_field_test->var.name) != 0)
5323 return false;
5324 }
5325
5326 for (i = 0; i < hist_data->n_sort_keys; i++) {
5327 sort_key = &hist_data->sort_keys[i];
5328 sort_key_test = &hist_data_test->sort_keys[i];
5329
5330 if (sort_key->field_idx != sort_key_test->field_idx ||
5331 sort_key->descending != sort_key_test->descending)
5332 return false;
5333 }
5334
5335 if (!ignore_filter && data->filter_str &&
5336 (strcmp(data->filter_str, data_test->filter_str) != 0))
5337 return false;
5338
5339 if (!actions_match(hist_data, hist_data_test))
5340 return false;
5341
5342 return true;
5343 }
5344
5345 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5346 struct event_trigger_data *data,
5347 struct trace_event_file *file)
5348 {
5349 struct hist_trigger_data *hist_data = data->private_data;
5350 struct event_trigger_data *test, *named_data = NULL;
5351 struct trace_array *tr = file->tr;
5352 int ret = 0;
5353
5354 if (hist_data->attrs->name) {
5355 named_data = find_named_trigger(hist_data->attrs->name);
5356 if (named_data) {
5357 if (!hist_trigger_match(data, named_data, named_data,
5358 true)) {
5359 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5360 ret = -EINVAL;
5361 goto out;
5362 }
5363 }
5364 }
5365
5366 if (hist_data->attrs->name && !named_data)
5367 goto new;
5368
5369 lockdep_assert_held(&event_mutex);
5370
5371 list_for_each_entry(test, &file->triggers, list) {
5372 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5373 if (!hist_trigger_match(data, test, named_data, false))
5374 continue;
5375 if (hist_data->attrs->pause)
5376 test->paused = true;
5377 else if (hist_data->attrs->cont)
5378 test->paused = false;
5379 else if (hist_data->attrs->clear)
5380 hist_clear(test);
5381 else {
5382 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5383 ret = -EEXIST;
5384 }
5385 goto out;
5386 }
5387 }
5388 new:
5389 if (hist_data->attrs->cont || hist_data->attrs->clear) {
5390 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5391 ret = -ENOENT;
5392 goto out;
5393 }
5394
5395 if (hist_data->attrs->pause)
5396 data->paused = true;
5397
5398 if (named_data) {
5399 data->private_data = named_data->private_data;
5400 set_named_trigger_data(data, named_data);
5401 data->ops = &event_hist_trigger_named_ops;
5402 }
5403
5404 if (data->ops->init) {
5405 ret = data->ops->init(data->ops, data);
5406 if (ret < 0)
5407 goto out;
5408 }
5409
5410 if (hist_data->enable_timestamps) {
5411 char *clock = hist_data->attrs->clock;
5412
5413 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5414 if (ret) {
5415 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5416 goto out;
5417 }
5418
5419 tracing_set_time_stamp_abs(file->tr, true);
5420 }
5421
5422 if (named_data)
5423 destroy_hist_data(hist_data);
5424
5425 ret++;
5426 out:
5427 return ret;
5428 }
5429
5430 static int hist_trigger_enable(struct event_trigger_data *data,
5431 struct trace_event_file *file)
5432 {
5433 int ret = 0;
5434
5435 list_add_tail_rcu(&data->list, &file->triggers);
5436
5437 update_cond_flag(file);
5438
5439 if (trace_event_trigger_enable_disable(file, 1) < 0) {
5440 list_del_rcu(&data->list);
5441 update_cond_flag(file);
5442 ret--;
5443 }
5444
5445 return ret;
5446 }
5447
5448 static bool have_hist_trigger_match(struct event_trigger_data *data,
5449 struct trace_event_file *file)
5450 {
5451 struct hist_trigger_data *hist_data = data->private_data;
5452 struct event_trigger_data *test, *named_data = NULL;
5453 bool match = false;
5454
5455 lockdep_assert_held(&event_mutex);
5456
5457 if (hist_data->attrs->name)
5458 named_data = find_named_trigger(hist_data->attrs->name);
5459
5460 list_for_each_entry(test, &file->triggers, list) {
5461 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5462 if (hist_trigger_match(data, test, named_data, false)) {
5463 match = true;
5464 break;
5465 }
5466 }
5467 }
5468
5469 return match;
5470 }
5471
5472 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5473 struct trace_event_file *file)
5474 {
5475 struct hist_trigger_data *hist_data = data->private_data;
5476 struct event_trigger_data *test, *named_data = NULL;
5477
5478 lockdep_assert_held(&event_mutex);
5479
5480 if (hist_data->attrs->name)
5481 named_data = find_named_trigger(hist_data->attrs->name);
5482
5483 list_for_each_entry(test, &file->triggers, list) {
5484 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5485 if (!hist_trigger_match(data, test, named_data, false))
5486 continue;
5487 hist_data = test->private_data;
5488 if (check_var_refs(hist_data))
5489 return true;
5490 break;
5491 }
5492 }
5493
5494 return false;
5495 }
5496
5497 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5498 struct event_trigger_data *data,
5499 struct trace_event_file *file)
5500 {
5501 struct hist_trigger_data *hist_data = data->private_data;
5502 struct event_trigger_data *test, *named_data = NULL;
5503 bool unregistered = false;
5504
5505 lockdep_assert_held(&event_mutex);
5506
5507 if (hist_data->attrs->name)
5508 named_data = find_named_trigger(hist_data->attrs->name);
5509
5510 list_for_each_entry(test, &file->triggers, list) {
5511 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5512 if (!hist_trigger_match(data, test, named_data, false))
5513 continue;
5514 unregistered = true;
5515 list_del_rcu(&test->list);
5516 trace_event_trigger_enable_disable(file, 0);
5517 update_cond_flag(file);
5518 break;
5519 }
5520 }
5521
5522 if (unregistered && test->ops->free)
5523 test->ops->free(test->ops, test);
5524
5525 if (hist_data->enable_timestamps) {
5526 if (!hist_data->remove || unregistered)
5527 tracing_set_time_stamp_abs(file->tr, false);
5528 }
5529 }
5530
5531 static bool hist_file_check_refs(struct trace_event_file *file)
5532 {
5533 struct hist_trigger_data *hist_data;
5534 struct event_trigger_data *test;
5535
5536 lockdep_assert_held(&event_mutex);
5537
5538 list_for_each_entry(test, &file->triggers, list) {
5539 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5540 hist_data = test->private_data;
5541 if (check_var_refs(hist_data))
5542 return true;
5543 }
5544 }
5545
5546 return false;
5547 }
5548
5549 static void hist_unreg_all(struct trace_event_file *file)
5550 {
5551 struct event_trigger_data *test, *n;
5552 struct hist_trigger_data *hist_data;
5553 struct synth_event *se;
5554 const char *se_name;
5555
5556 lockdep_assert_held(&event_mutex);
5557
5558 if (hist_file_check_refs(file))
5559 return;
5560
5561 list_for_each_entry_safe(test, n, &file->triggers, list) {
5562 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5563 hist_data = test->private_data;
5564 list_del_rcu(&test->list);
5565 trace_event_trigger_enable_disable(file, 0);
5566
5567 se_name = trace_event_name(file->event_call);
5568 se = find_synth_event(se_name);
5569 if (se)
5570 se->ref--;
5571
5572 update_cond_flag(file);
5573 if (hist_data->enable_timestamps)
5574 tracing_set_time_stamp_abs(file->tr, false);
5575 if (test->ops->free)
5576 test->ops->free(test->ops, test);
5577 }
5578 }
5579 }
5580
5581 static int event_hist_trigger_func(struct event_command *cmd_ops,
5582 struct trace_event_file *file,
5583 char *glob, char *cmd, char *param)
5584 {
5585 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5586 struct event_trigger_data *trigger_data;
5587 struct hist_trigger_attrs *attrs;
5588 struct event_trigger_ops *trigger_ops;
5589 struct hist_trigger_data *hist_data;
5590 struct synth_event *se;
5591 const char *se_name;
5592 bool remove = false;
5593 char *trigger, *p;
5594 int ret = 0;
5595
5596 lockdep_assert_held(&event_mutex);
5597
5598 if (glob && strlen(glob)) {
5599 hist_err_clear();
5600 last_cmd_set(file, param);
5601 }
5602
5603 if (!param)
5604 return -EINVAL;
5605
5606 if (glob[0] == '!')
5607 remove = true;
5608
5609 /*
5610 * separate the trigger from the filter (k:v [if filter])
5611 * allowing for whitespace in the trigger
5612 */
5613 p = trigger = param;
5614 do {
5615 p = strstr(p, "if");
5616 if (!p)
5617 break;
5618 if (p == param)
5619 return -EINVAL;
5620 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5621 p++;
5622 continue;
5623 }
5624 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5625 return -EINVAL;
5626 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5627 p++;
5628 continue;
5629 }
5630 break;
5631 } while (p);
5632
5633 if (!p)
5634 param = NULL;
5635 else {
5636 *(p - 1) = '\0';
5637 param = strstrip(p);
5638 trigger = strstrip(trigger);
5639 }
5640
5641 attrs = parse_hist_trigger_attrs(file->tr, trigger);
5642 if (IS_ERR(attrs))
5643 return PTR_ERR(attrs);
5644
5645 if (attrs->map_bits)
5646 hist_trigger_bits = attrs->map_bits;
5647
5648 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5649 if (IS_ERR(hist_data)) {
5650 destroy_hist_trigger_attrs(attrs);
5651 return PTR_ERR(hist_data);
5652 }
5653
5654 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5655
5656 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5657 if (!trigger_data) {
5658 ret = -ENOMEM;
5659 goto out_free;
5660 }
5661
5662 trigger_data->count = -1;
5663 trigger_data->ops = trigger_ops;
5664 trigger_data->cmd_ops = cmd_ops;
5665
5666 INIT_LIST_HEAD(&trigger_data->list);
5667 RCU_INIT_POINTER(trigger_data->filter, NULL);
5668
5669 trigger_data->private_data = hist_data;
5670
5671 /* if param is non-empty, it's supposed to be a filter */
5672 if (param && cmd_ops->set_filter) {
5673 ret = cmd_ops->set_filter(param, trigger_data, file);
5674 if (ret < 0)
5675 goto out_free;
5676 }
5677
5678 if (remove) {
5679 if (!have_hist_trigger_match(trigger_data, file))
5680 goto out_free;
5681
5682 if (hist_trigger_check_refs(trigger_data, file)) {
5683 ret = -EBUSY;
5684 goto out_free;
5685 }
5686
5687 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5688 se_name = trace_event_name(file->event_call);
5689 se = find_synth_event(se_name);
5690 if (se)
5691 se->ref--;
5692 ret = 0;
5693 goto out_free;
5694 }
5695
5696 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5697 /*
5698 * The above returns on success the # of triggers registered,
5699 * but if it didn't register any it returns zero. Consider no
5700 * triggers registered a failure too.
5701 */
5702 if (!ret) {
5703 if (!(attrs->pause || attrs->cont || attrs->clear))
5704 ret = -ENOENT;
5705 goto out_free;
5706 } else if (ret < 0)
5707 goto out_free;
5708
5709 if (get_named_trigger_data(trigger_data))
5710 goto enable;
5711
5712 if (has_hist_vars(hist_data))
5713 save_hist_vars(hist_data);
5714
5715 ret = create_actions(hist_data);
5716 if (ret)
5717 goto out_unreg;
5718
5719 ret = tracing_map_init(hist_data->map);
5720 if (ret)
5721 goto out_unreg;
5722 enable:
5723 ret = hist_trigger_enable(trigger_data, file);
5724 if (ret)
5725 goto out_unreg;
5726
5727 se_name = trace_event_name(file->event_call);
5728 se = find_synth_event(se_name);
5729 if (se)
5730 se->ref++;
5731 /* Just return zero, not the number of registered triggers */
5732 ret = 0;
5733 out:
5734 if (ret == 0)
5735 hist_err_clear();
5736
5737 return ret;
5738 out_unreg:
5739 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5740 out_free:
5741 if (cmd_ops->set_filter)
5742 cmd_ops->set_filter(NULL, trigger_data, NULL);
5743
5744 remove_hist_vars(hist_data);
5745
5746 kfree(trigger_data);
5747
5748 destroy_hist_data(hist_data);
5749 goto out;
5750 }
5751
5752 static struct event_command trigger_hist_cmd = {
5753 .name = "hist",
5754 .trigger_type = ETT_EVENT_HIST,
5755 .flags = EVENT_CMD_FL_NEEDS_REC,
5756 .func = event_hist_trigger_func,
5757 .reg = hist_register_trigger,
5758 .unreg = hist_unregister_trigger,
5759 .unreg_all = hist_unreg_all,
5760 .get_trigger_ops = event_hist_get_trigger_ops,
5761 .set_filter = set_trigger_filter,
5762 };
5763
5764 __init int register_trigger_hist_cmd(void)
5765 {
5766 int ret;
5767
5768 ret = register_event_command(&trigger_hist_cmd);
5769 WARN_ON(ret < 0);
5770
5771 return ret;
5772 }
5773
5774 static void
5775 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5776 struct ring_buffer_event *event)
5777 {
5778 struct enable_trigger_data *enable_data = data->private_data;
5779 struct event_trigger_data *test;
5780
5781 list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5782 lockdep_is_held(&event_mutex)) {
5783 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5784 if (enable_data->enable)
5785 test->paused = false;
5786 else
5787 test->paused = true;
5788 }
5789 }
5790 }
5791
5792 static void
5793 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5794 struct ring_buffer_event *event)
5795 {
5796 if (!data->count)
5797 return;
5798
5799 if (data->count != -1)
5800 (data->count)--;
5801
5802 hist_enable_trigger(data, rec, event);
5803 }
5804
5805 static struct event_trigger_ops hist_enable_trigger_ops = {
5806 .func = hist_enable_trigger,
5807 .print = event_enable_trigger_print,
5808 .init = event_trigger_init,
5809 .free = event_enable_trigger_free,
5810 };
5811
5812 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5813 .func = hist_enable_count_trigger,
5814 .print = event_enable_trigger_print,
5815 .init = event_trigger_init,
5816 .free = event_enable_trigger_free,
5817 };
5818
5819 static struct event_trigger_ops hist_disable_trigger_ops = {
5820 .func = hist_enable_trigger,
5821 .print = event_enable_trigger_print,
5822 .init = event_trigger_init,
5823 .free = event_enable_trigger_free,
5824 };
5825
5826 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5827 .func = hist_enable_count_trigger,
5828 .print = event_enable_trigger_print,
5829 .init = event_trigger_init,
5830 .free = event_enable_trigger_free,
5831 };
5832
5833 static struct event_trigger_ops *
5834 hist_enable_get_trigger_ops(char *cmd, char *param)
5835 {
5836 struct event_trigger_ops *ops;
5837 bool enable;
5838
5839 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5840
5841 if (enable)
5842 ops = param ? &hist_enable_count_trigger_ops :
5843 &hist_enable_trigger_ops;
5844 else
5845 ops = param ? &hist_disable_count_trigger_ops :
5846 &hist_disable_trigger_ops;
5847
5848 return ops;
5849 }
5850
5851 static void hist_enable_unreg_all(struct trace_event_file *file)
5852 {
5853 struct event_trigger_data *test, *n;
5854
5855 list_for_each_entry_safe(test, n, &file->triggers, list) {
5856 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5857 list_del_rcu(&test->list);
5858 update_cond_flag(file);
5859 trace_event_trigger_enable_disable(file, 0);
5860 if (test->ops->free)
5861 test->ops->free(test->ops, test);
5862 }
5863 }
5864 }
5865
5866 static struct event_command trigger_hist_enable_cmd = {
5867 .name = ENABLE_HIST_STR,
5868 .trigger_type = ETT_HIST_ENABLE,
5869 .func = event_enable_trigger_func,
5870 .reg = event_enable_register_trigger,
5871 .unreg = event_enable_unregister_trigger,
5872 .unreg_all = hist_enable_unreg_all,
5873 .get_trigger_ops = hist_enable_get_trigger_ops,
5874 .set_filter = set_trigger_filter,
5875 };
5876
5877 static struct event_command trigger_hist_disable_cmd = {
5878 .name = DISABLE_HIST_STR,
5879 .trigger_type = ETT_HIST_ENABLE,
5880 .func = event_enable_trigger_func,
5881 .reg = event_enable_register_trigger,
5882 .unreg = event_enable_unregister_trigger,
5883 .unreg_all = hist_enable_unreg_all,
5884 .get_trigger_ops = hist_enable_get_trigger_ops,
5885 .set_filter = set_trigger_filter,
5886 };
5887
5888 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5889 {
5890 unregister_event_command(&trigger_hist_enable_cmd);
5891 unregister_event_command(&trigger_hist_disable_cmd);
5892 }
5893
5894 __init int register_trigger_hist_enable_disable_cmds(void)
5895 {
5896 int ret;
5897
5898 ret = register_event_command(&trigger_hist_enable_cmd);
5899 if (WARN_ON(ret < 0))
5900 return ret;
5901 ret = register_event_command(&trigger_hist_disable_cmd);
5902 if (WARN_ON(ret < 0))
5903 unregister_trigger_hist_enable_disable_cmds();
5904
5905 return ret;
5906 }
Linux with added FPC-III support