6 - arch/s390/kernel/debug.c
7 - arch/s390/include/asm/debug.h
11 The goal of this feature is to provide a kernel debug logging API
12 where log records can be stored efficiently in memory, where each component
13 (e.g. device drivers) can have one separate debug log.
14 One purpose of this is to inspect the debug logs after a production system crash
15 in order to analyze the reason for the crash.
17 If the system still runs but only a subcomponent which uses dbf fails,
18 it is possible to look at the debug logs on a live system via the Linux
21 The debug feature may also very useful for kernel and driver development.
25 Kernel components (e.g. device drivers) can register themselves at the debug
26 feature with the function call :c:func:`debug_register()`.
27 This function initializes a
28 debug log for the caller. For each debug log exists a number of debug areas
29 where exactly one is active at one time. Each debug area consists of contiguous
30 pages in memory. In the debug areas there are stored debug entries (log records)
31 which are written by event- and exception-calls.
33 An event-call writes the specified debug entry to the active debug
34 area and updates the log pointer for the active area. If the end
35 of the active debug area is reached, a wrap around is done (ring buffer)
36 and the next debug entry will be written at the beginning of the active
39 An exception-call writes the specified debug entry to the log and
40 switches to the next debug area. This is done in order to be sure
41 that the records which describe the origin of the exception are not
42 overwritten when a wrap around for the current area occurs.
44 The debug areas themselves are also ordered in form of a ring buffer.
45 When an exception is thrown in the last debug area, the following debug
46 entries are then written again in the very first area.
48 There are four versions for the event- and exception-calls: One for
49 logging raw data, one for text, one for numbers (unsigned int and long),
50 and one for sprintf-like formatted strings.
52 Each debug entry contains the following data:
55 - Cpu-Number of calling task
56 - Level of debug entry (0...6)
57 - Return Address to caller
58 - Flag, if entry is an exception or not
60 The debug logs can be inspected in a live system through entries in
61 the debugfs-filesystem. Under the toplevel directory "``s390dbf``" there is
62 a directory for each registered component, which is named like the
63 corresponding component. The debugfs normally should be mounted to
64 ``/sys/kernel/debug`` therefore the debug feature can be accessed under
65 ``/sys/kernel/debug/s390dbf``.
67 The content of the directories are files which represent different views
68 to the debug log. Each component can decide which views should be
69 used through registering them with the function :c:func:`debug_register_view()`.
70 Predefined views for hex/ascii, sprintf and raw binary data are provided.
71 It is also possible to define other views. The content of
72 a view can be inspected simply by reading the corresponding debugfs file.
74 All debug logs have an actual debug level (range from 0 to 6).
75 The default level is 3. Event and Exception functions have a :c:data:`level`
76 parameter. Only debug entries with a level that is lower or equal
77 than the actual level are written to the log. This means, when
78 writing events, high priority log entries should have a low level
79 value whereas low priority entries should have a high one.
80 The actual debug level can be changed with the help of the debugfs-filesystem
81 through writing a number string "x" to the ``level`` debugfs file which is
82 provided for every debug log. Debugging can be switched off completely
83 by using "-" on the ``level`` debugfs file.
87 > echo "-" > /sys/kernel/debug/s390dbf/dasd/level
89 It is also possible to deactivate the debug feature globally for every
90 debug log. You can change the behavior using 2 sysctl parameters in
91 ``/proc/sys/s390dbf``:
93 There are currently 2 possible triggers, which stop the debug feature
94 globally. The first possibility is to use the ``debug_active`` sysctl. If
95 set to 1 the debug feature is running. If ``debug_active`` is set to 0 the
96 debug feature is turned off.
98 The second trigger which stops the debug feature is a kernel oops.
99 That prevents the debug feature from overwriting debug information that
100 happened before the oops. After an oops you can reactivate the debug feature
101 by piping 1 to ``/proc/sys/s390dbf/debug_active``. Nevertheless, it's not
102 suggested to use an oopsed kernel in a production environment.
104 If you want to disallow the deactivation of the debug feature, you can use
105 the ``debug_stoppable`` sysctl. If you set ``debug_stoppable`` to 0 the debug
106 feature cannot be stopped. If the debug feature is already stopped, it
107 will stay deactivated.
112 .. kernel-doc:: arch/s390/kernel/debug.c
113 .. kernel-doc:: arch/s390/include/asm/debug.h
120 extern struct debug_view debug_hex_ascii_view;
122 extern struct debug_view debug_raw_view;
124 extern struct debug_view debug_sprintf_view;
132 * hex_ascii- + raw-view Example
135 #include <linux/init.h>
136 #include <asm/debug.h>
138 static debug_info_t *debug_info;
140 static int init(void)
142 /* register 4 debug areas with one page each and 4 byte data field */
144 debug_info = debug_register("test", 1, 4, 4 );
145 debug_register_view(debug_info, &debug_hex_ascii_view);
146 debug_register_view(debug_info, &debug_raw_view);
148 debug_text_event(debug_info, 4 , "one ");
149 debug_int_exception(debug_info, 4, 4711);
150 debug_event(debug_info, 3, &debug_info, 4);
155 static void cleanup(void)
157 debug_unregister(debug_info);
161 module_exit(cleanup);
166 * sprintf-view Example
169 #include <linux/init.h>
170 #include <asm/debug.h>
172 static debug_info_t *debug_info;
174 static int init(void)
176 /* register 4 debug areas with one page each and data field for */
177 /* format string pointer + 2 varargs (= 3 * sizeof(long)) */
179 debug_info = debug_register("test", 1, 4, sizeof(long) * 3);
180 debug_register_view(debug_info, &debug_sprintf_view);
182 debug_sprintf_event(debug_info, 2 , "first event in %s:%i\n",__FILE__,__LINE__);
183 debug_sprintf_exception(debug_info, 1, "pointer to debug info: %p\n",&debug_info);
188 static void cleanup(void)
190 debug_unregister(debug_info);
194 module_exit(cleanup);
198 Views to the debug logs can be investigated through reading the corresponding
203 > ls /sys/kernel/debug/s390dbf/dasd
204 flush hex_ascii level pages raw
205 > cat /sys/kernel/debug/s390dbf/dasd/hex_ascii | sort -k2,2 -s
206 00 00974733272:680099 2 - 02 0006ad7e 07 ea 4a 90 | ....
207 00 00974733272:682210 2 - 02 0006ade6 46 52 45 45 | FREE
208 00 00974733272:682213 2 - 02 0006adf6 07 ea 4a 90 | ....
209 00 00974733272:682281 1 * 02 0006ab08 41 4c 4c 43 | EXCP
210 01 00974733272:682284 2 - 02 0006ab16 45 43 4b 44 | ECKD
211 01 00974733272:682287 2 - 02 0006ab28 00 00 00 04 | ....
212 01 00974733272:682289 2 - 02 0006ab3e 00 00 00 20 | ...
213 01 00974733272:682297 2 - 02 0006ad7e 07 ea 4a 90 | ....
214 01 00974733272:684384 2 - 00 0006ade6 46 52 45 45 | FREE
215 01 00974733272:684388 2 - 00 0006adf6 07 ea 4a 90 | ....
217 See section about predefined views for explanation of the above output!
219 Changing the debug level
220 ------------------------
225 > cat /sys/kernel/debug/s390dbf/dasd/level
227 > echo "5" > /sys/kernel/debug/s390dbf/dasd/level
228 > cat /sys/kernel/debug/s390dbf/dasd/level
233 Debug areas can be flushed with piping the number of the desired
234 area (0...n) to the debugfs file "flush". When using "-" all debug areas
239 1. Flush debug area 0::
241 > echo "0" > /sys/kernel/debug/s390dbf/dasd/flush
243 2. Flush all debug areas::
245 > echo "-" > /sys/kernel/debug/s390dbf/dasd/flush
247 Changing the size of debug areas
248 ------------------------------------
249 It is possible the change the size of debug areas through piping
250 the number of pages to the debugfs file "pages". The resize request will
251 also flush the debug areas.
255 Define 4 pages for the debug areas of debug feature "dasd"::
257 > echo "4" > /sys/kernel/debug/s390dbf/dasd/pages
259 Stopping the debug feature
260 --------------------------
263 1. Check if stopping is allowed::
265 > cat /proc/sys/s390dbf/debug_stoppable
267 2. Stop debug feature::
269 > echo 0 > /proc/sys/s390dbf/debug_active
273 The ``crash`` tool since v5.1.0 has a built-in command
274 ``s390dbf`` to display all the debug logs or export them to the file system.
275 With this tool it is possible
276 to investigate the debug logs on a live system and with a memory dump after
279 Investigating raw memory
280 ------------------------
281 One last possibility to investigate the debug logs at a live
282 system and after a system crash is to look at the raw memory
283 under VM or at the Service Element.
284 It is possible to find the anchor of the debug-logs through
285 the ``debug_area_first`` symbol in the System map. Then one has
286 to follow the correct pointers of the data-structures defined
287 in debug.h and find the debug-areas in memory.
288 Normally modules which use the debug feature will also have
289 a global variable with the pointer to the debug-logs. Following
290 this pointer it will also be possible to find the debug logs in
293 For this method it is recommended to use '16 * x + 4' byte (x = 0..n)
294 for the length of the data field in :c:func:`debug_register()` in
295 order to see the debug entries well formatted.
301 There are three predefined views: hex_ascii, raw and sprintf.
302 The hex_ascii view shows the data field in hex and ascii representation
303 (e.g. ``45 43 4b 44 | ECKD``).
304 The raw view returns a bytestream as the debug areas are stored in memory.
306 The sprintf view formats the debug entries in the same way as the sprintf
307 function would do. The sprintf event/exception functions write to the
308 debug entry a pointer to the format string (size = sizeof(long))
309 and for each vararg a long value. So e.g. for a debug entry with a format
310 string plus two varargs one would need to allocate a (3 * sizeof(long))
311 byte data area in the debug_register() function.
314 Using "%s" in sprintf event functions is dangerous. You can only
315 use "%s" in the sprintf event functions, if the memory for the passed string
316 is available as long as the debug feature exists. The reason behind this is
317 that due to performance considerations only a pointer to the string is stored
318 in the debug feature. If you log a string that is freed afterwards, you will
319 get an OOPS when inspecting the debug feature, because then the debug feature
320 will access the already freed memory.
323 If using the sprintf view do NOT use other event/exception functions
324 than the sprintf-event and -exception functions.
326 The format of the hex_ascii and sprintf view is as follows:
329 - Timestamp (formatted as seconds and microseconds since 00:00:00 Coordinated
330 Universal Time (UTC), January 1, 1970)
331 - level of debug entry
332 - Exception flag (* = Exception)
333 - Cpu-Number of calling task
334 - Return Address to caller
337 The format of the raw view is:
339 - Header as described in debug.h
342 A typical line of the hex_ascii view will look like the following (first line
343 is only for explanation and will not be displayed when 'cating' the view)::
345 area time level exception cpu caller data (hex + ascii)
346 --------------------------------------------------------------------------
347 00 00964419409:440690 1 - 00 88023fe
353 Views are specified with the 'debug_view' structure. There are defined
354 callback functions which are used for reading and writing the debugfs files:
359 char name[DEBUG_MAX_PROCF_LEN];
360 debug_prolog_proc_t* prolog_proc;
361 debug_header_proc_t* header_proc;
362 debug_format_proc_t* format_proc;
363 debug_input_proc_t* input_proc;
371 typedef int (debug_header_proc_t) (debug_info_t* id,
372 struct debug_view* view,
374 debug_entry_t* entry,
377 typedef int (debug_format_proc_t) (debug_info_t* id,
378 struct debug_view* view, char* out_buf,
380 typedef int (debug_prolog_proc_t) (debug_info_t* id,
381 struct debug_view* view,
383 typedef int (debug_input_proc_t) (debug_info_t* id,
384 struct debug_view* view,
385 struct file* file, const char* user_buf,
386 size_t in_buf_size, loff_t* offset);
389 The "private_data" member can be used as pointer to view specific data.
390 It is not used by the debug feature itself.
392 The output when reading a debugfs file is structured like this::
396 "header_proc output 1" "format_proc output 1"
397 "header_proc output 2" "format_proc output 2"
398 "header_proc output 3" "format_proc output 3"
401 When a view is read from the debugfs, the Debug Feature calls the
402 'prolog_proc' once for writing the prolog.
403 Then 'header_proc' and 'format_proc' are called for each
404 existing debug entry.
406 The input_proc can be used to implement functionality when it is written to
407 the view (e.g. like with ``echo "0" > /sys/kernel/debug/s390dbf/dasd/level``).
409 For header_proc there can be used the default function
410 :c:func:`debug_dflt_header_fn()` which is defined in debug.h.
411 and which produces the same header output as the predefined views.
414 00 00964419409:440761 2 - 00 88023ec
416 In order to see how to use the callback functions check the implementation
417 of the default views!
423 #include <asm/debug.h>
425 #define UNKNOWNSTR "data: %08x"
427 const char* messages[] =
428 {"This error...........\n",
429 "That error...........\n",
430 "Problem..............\n",
431 "Something went wrong.\n",
432 "Everything ok........\n",
436 static int debug_test_format_fn(
437 debug_info_t *id, struct debug_view *view,
438 char *out_buf, const char *in_buf
443 if (id->buf_size >= 4) {
444 int msg_nr = *((int*)in_buf);
445 if (msg_nr < sizeof(messages) / sizeof(char*) - 1)
446 rc += sprintf(out_buf, "%s", messages[msg_nr]);
448 rc += sprintf(out_buf, UNKNOWNSTR, msg_nr);
453 struct debug_view debug_test_view = {
454 "myview", /* name of view */
455 NULL, /* no prolog */
456 &debug_dflt_header_fn, /* default header for each entry */
457 &debug_test_format_fn, /* our own format function */
458 NULL, /* no input function */
459 NULL /* no private data */
467 debug_info_t *debug_info;
470 debug_info = debug_register("test", 0, 4, 4);
471 debug_register_view(debug_info, &debug_test_view);
472 for (i = 0; i < 10; i ++)
473 debug_int_event(debug_info, 1, i);
477 > cat /sys/kernel/debug/s390dbf/test/myview
478 00 00964419734:611402 1 - 00 88042ca This error...........
479 00 00964419734:611405 1 - 00 88042ca That error...........
480 00 00964419734:611408 1 - 00 88042ca Problem..............
481 00 00964419734:611411 1 - 00 88042ca Something went wrong.
482 00 00964419734:611414 1 - 00 88042ca Everything ok........
483 00 00964419734:611417 1 - 00 88042ca data: 00000005
484 00 00964419734:611419 1 - 00 88042ca data: 00000006
485 00 00964419734:611422 1 - 00 88042ca data: 00000007
486 00 00964419734:611425 1 - 00 88042ca data: 00000008
487 00 00964419734:611428 1 - 00 88042ca data: 00000009