Commit 5cf9c84e authored by Arnaldo Carvalho de Melo's avatar Arnaldo Carvalho de Melo
Browse files

perf trace: Introduce --min-stack filter

Counterpart to --max-stack, to help focusing on deeply nested calls. Can
be combined with --duration, etc.

E.g.:

  System wide syscall tracing looking for call stacks longer than 66:

  # trace --mmap-pages 32768 --filter-pid 2711 --call-graph dwarf,16384 --min-stack 66

  Or more compactly:

  # trace -m 32768 --filt 2711 --call dwarf,16384 --min-st 66
   363.027 ( 0.002 ms): gnome-shell/2287 poll(ufds: 0x7ffc5ea24230, nfds: 1, timeout_msecs: 4294967295         ) = 1
                                       [0xf6fdd] (/usr/lib64/libc-2.22.so)
                                       _xcb_conn_wait+0x92 (/usr/lib64/libxcb.so.1.1.0)
                                       _xcb_out_send+0x4d (/usr/lib64/libxcb.so.1.1.0)
                                       xcb_writev+0x45 (/usr/lib64/libxcb.so.1.1.0)
                                       _XSend+0x19e (/usr/lib64/libX11.so.6.3.0)
                                       _XReply+0x82 (/usr/lib64/libX11.so.6.3.0)
                                       XSync+0x4d (/usr/lib64/libX11.so.6.3.0)
                                       dri3_bind_tex_image+0x42 (/usr/lib64/libGL.so.1.2.0)
                                       _cogl_winsys_texture_pixmap_x11_update+0x117 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_texture_pixmap_x11_update+0x67 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_texture_pixmap_x11_pre_paint+0x13 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_pipeline_layer_pre_paint+0x5e (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_rectangles_validate_layer_cb+0x1b (/usr/lib64/libcogl.so.20.4.1)
                                       cogl_pipeline_foreach_layer+0xbe (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_framebuffer_draw_multitextured_rectangles+0x77 (/usr/lib64/libcogl.so.20.4.1)
                                       cogl_framebuffer_draw_multitextured_rectangle+0x51 (/usr/lib64/libcogl.so.20.4.1)
                                       paint_clipped_rectangle+0xb6 (/usr/lib64/libmutter.so.0.0.0)
                                       meta_shaped_texture_paint+0x3e3 (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_window_actor_paint+0x14b (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_window_group_paint+0x19f (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       [0x3d970] (/usr/lib64/gnome-shell/libgnome-shell.so)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_stage_paint+0x3a (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_stage_paint+0x45 (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0x164 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _clutter_stage_do_paint+0x17b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_stage_cogl_redraw+0x496 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _clutter_stage_do_update+0x117 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_clock_dispatch+0x169 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       g_main_context_dispatch+0x15a (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       g_main_context_iterate.isra.29+0x1e0 (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       g_main_loop_run+0xc2 (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       meta_run+0x2c (/usr/lib64/libmutter.so.0.0.0)
                                       main+0x3f7 (/usr/bin/gnome-shell)
                                       __libc_start_main+0xf0 (/usr/lib64/libc-2.22.so)
                                       [0x2909] (/usr/bin/gnome-shell)
   363.038 ( 0.006 ms): gnome-shell/2287 writev(fd: 5<socket:[32540]>, vec: 0x7ffc5ea243a0, vlen: 3            ) = 4
                                       __GI___writev+0x2d (/usr/lib64/libc-2.22.so)
                                       _xcb_conn_wait+0x359 (/usr/lib64/libxcb.so.1.1.0)
                                       _xcb_out_send+0x4d (/usr/lib64/libxcb.so.1.1.0)
                                       xcb_writev+0x45 (/usr/lib64/libxcb.so.1.1.0)
                                       _XSend+0x19e (/usr/lib64/libX11.so.6.3.0)
                                       _XReply+0x82 (/usr/lib64/libX11.so.6.3.0)
                                       XSync+0x4d (/usr/lib64/libX11.so.6.3.0)
                                       dri3_bind_tex_image+0x42 (/usr/lib64/libGL.so.1.2.0)
                                       _cogl_winsys_texture_pixmap_x11_update+0x117 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_texture_pixmap_x11_update+0x67 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_texture_pixmap_x11_pre_paint+0x13 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_pipeline_layer_pre_paint+0x5e (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_rectangles_validate_layer_cb+0x1b (/usr/lib64/libcogl.so.20.4.1)
                                       cogl_pipeline_foreach_layer+0xbe (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_framebuffer_draw_multitextured_rectangles+0x77 (/usr/lib64/libcogl.so.20.4.1)
                                       cogl_framebuffer_draw_multitextured_rectangle+0x51 (/usr/lib64/libcogl.so.20.4.1)
                                       paint_clipped_rectangle+0xb6 (/usr/lib64/libmutter.so.0.0.0)
                                       meta_shaped_texture_paint+0x3e3 (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_window_actor_paint+0x14b (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_window_group_paint+0x19f (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       [0x3d970] (/usr/lib64/gnome-shell/libgnome-shell.so)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_stage_paint+0x3a (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_stage_paint+0x45 (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0x164 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _clutter_stage_do_paint+0x17b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_stage_cogl_redraw+0x496 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _clutter_stage_do_update+0x117 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_clock_dispatch+0x169 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       g_main_context_dispatch+0x15a (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       g_main_context_iterate.isra.29+0x1e0 (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       g_main_loop_run+0xc2 (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       meta_run+0x2c (/usr/lib64/libmutter.so.0.0.0)
                                       main+0x3f7 (/usr/bin/gnome-shell)
                                       __libc_start_main+0xf0 (/usr/lib64/libc-2.22.so)
                                       [0x2909] (/usr/bin/gnome-shell)
   363.086 ( 0.042 ms): gnome-shell/2287 poll(ufds: 0x7ffc5ea24250, nfds: 1, timeout_msecs: 4294967295         ) = 1
                                       [0xf6fdd] (/usr/lib64/libc-2.22.so)
                                       _xcb_conn_wait+0x92 (/usr/lib64/libxcb.so.1.1.0)
                                       wait_for_reply+0xb7 (/usr/lib64/libxcb.so.1.1.0)
                                       xcb_wait_for_reply+0x61 (/usr/lib64/libxcb.so.1.1.0)
                                       _XReply+0x127 (/usr/lib64/libX11.so.6.3.0)
                                       XSync+0x4d (/usr/lib64/libX11.so.6.3.0)
                                       dri3_bind_tex_image+0x42 (/usr/lib64/libGL.so.1.2.0)
                                       _cogl_winsys_texture_pixmap_x11_update+0x117 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_texture_pixmap_x11_update+0x67 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_texture_pixmap_x11_pre_paint+0x13 (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_pipeline_layer_pre_paint+0x5e (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_rectangles_validate_layer_cb+0x1b (/usr/lib64/libcogl.so.20.4.1)
                                       cogl_pipeline_foreach_layer+0xbe (/usr/lib64/libcogl.so.20.4.1)
                                       _cogl_framebuffer_draw_multitextured_rectangles+0x77 (/usr/lib64/libcogl.so.20.4.1)
                                       cogl_framebuffer_draw_multitextured_rectangle+0x51 (/usr/lib64/libcogl.so.20.4.1)
                                       paint_clipped_rectangle+0xb6 (/usr/lib64/libmutter.so.0.0.0)
                                       meta_shaped_texture_paint+0x3e3 (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_window_actor_paint+0x14b (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_real_paint+0x20 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_window_group_paint+0x19f (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       [0x3d970] (/usr/lib64/gnome-shell/libgnome-shell.so)
                                       _g_closure_invoke_va+0xb2 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_stage_paint+0x3a (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       meta_stage_paint+0x45 (/usr/lib64/libmutter.so.0.0.0)
                                       _g_closure_invoke_va+0x164 (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit_valist+0xc0d (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       g_signal_emit+0x8f (/usr/lib64/libgobject-2.0.so.0.4600.2)
                                       clutter_actor_continue_paint+0x2bb (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_actor_paint.part.41+0x47b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _clutter_stage_do_paint+0x17b (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_stage_cogl_redraw+0x496 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       _clutter_stage_do_update+0x117 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       clutter_clock_dispatch+0x169 (/usr/lib64/libclutter-1.0.so.0.2400.2)
                                       g_main_context_dispatch+0x15a (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       g_main_context_iterate.isra.29+0x1e0 (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       g_main_loop_run+0xc2 (/usr/lib64/libglib-2.0.so.0.4600.2)
                                       meta_run+0x2c (/usr/lib64/libmutter.so.0.0.0)
                                       main+0x3f7 (/usr/bin/gnome-shell)
                                       __libc_start_main+0xf0 (/usr/lib64/libc-2.22.so)
                                       [0x2909] (/usr/bin/gnome-shell)

Cc: Adrian Hunter <adrian.hunter@intel.com>
Cc: David Ahern <dsahern@gmail.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Milian Wolff <milian.wolff@kdab.com>
Cc: Namhyung Kim <namhyung@kernel.org>
Cc: Wang Nan <wangnan0@huawei.com>
Link: http://lkml.kernel.org/n/tip-jncuxju9fibq2rl6olhqwjw6@git.kernel.org


Signed-off-by: default avatarArnaldo Carvalho de Melo <acme@redhat.com>
parent e519bd9a
Loading
Loading
Loading
Loading
+4 −0
Original line number Diff line number Diff line
@@ -138,6 +138,10 @@ the thread executes on the designated CPUs. Default is to monitor all CPUs.

        Default: 127

--min-stack::
        Set the stack depth limit when parsing the callchain, anything
        below the specified depth will be ignored. Disabled by default.

--proc-map-timeout::
	When processing pre-existing threads /proc/XXX/mmap, it may take a long time,
	because the file may be huge. A time out is needed in such cases.
+37 −18
Original line number Diff line number Diff line
@@ -108,6 +108,7 @@ struct trace {
				proc_getname;
	} stats;
	unsigned int		max_stack;
	unsigned int		min_stack;
	bool			not_ev_qualifier;
	bool			live;
	bool			full_time;
@@ -1849,7 +1850,7 @@ static int trace__sys_enter(struct trace *trace, struct perf_evsel *evsel,
			goto out_put;
	}

	if (!(trace->duration_filter || trace->summary_only))
	if (!(trace->duration_filter || trace->summary_only || trace->min_stack))
		trace__printf_interrupted_entry(trace, sample);

	ttrace->entry_time = sample->time;
@@ -1860,7 +1861,7 @@ static int trace__sys_enter(struct trace *trace, struct perf_evsel *evsel,
					   args, trace, thread);

	if (sc->is_exit) {
		if (!trace->duration_filter && !trace->summary_only) {
		if (!(trace->duration_filter || trace->summary_only || trace->min_stack)) {
			trace__fprintf_entry_head(trace, thread, 1, sample->time, trace->output);
			fprintf(trace->output, "%-70s\n", ttrace->entry_str);
		}
@@ -1880,26 +1881,26 @@ out_put:
	return err;
}

static int trace__fprintf_callchain(struct trace *trace, struct perf_evsel *evsel,
				    struct perf_sample *sample)
static int trace__resolve_callchain(struct trace *trace, struct perf_evsel *evsel,
				    struct perf_sample *sample,
				    struct callchain_cursor *cursor)
{
	struct addr_location al;
	/* TODO: user-configurable print_opts */
	const unsigned int print_opts = EVSEL__PRINT_SYM |
				        EVSEL__PRINT_DSO |
				        EVSEL__PRINT_UNKNOWN_AS_ADDR;

	if (sample->callchain == NULL)
		return 0;

	if (machine__resolve(trace->host, &al, sample) < 0 ||
	    thread__resolve_callchain(al.thread, &callchain_cursor, evsel,
				      sample, NULL, NULL, trace->max_stack)) {
		pr_err("Problem processing %s callchain, skipping...\n",
			perf_evsel__name(evsel));
	    thread__resolve_callchain(al.thread, cursor, evsel, sample, NULL, NULL, trace->max_stack))
		return -1;

	return 0;
}

static int trace__fprintf_callchain(struct trace *trace, struct perf_sample *sample)
{
	/* TODO: user-configurable print_opts */
	const unsigned int print_opts = EVSEL__PRINT_SYM |
				        EVSEL__PRINT_DSO |
				        EVSEL__PRINT_UNKNOWN_AS_ADDR;

	return sample__fprintf_callchain(sample, 38, print_opts, &callchain_cursor, trace->output);
}

@@ -1910,7 +1911,7 @@ static int trace__sys_exit(struct trace *trace, struct perf_evsel *evsel,
	long ret;
	u64 duration = 0;
	struct thread *thread;
	int id = perf_evsel__sc_tp_uint(evsel, id, sample), err = -1;
	int id = perf_evsel__sc_tp_uint(evsel, id, sample), err = -1, callchain_ret = 0;
	struct syscall *sc = trace__syscall_info(trace, evsel, id);
	struct thread_trace *ttrace;

@@ -1942,6 +1943,15 @@ static int trace__sys_exit(struct trace *trace, struct perf_evsel *evsel,
	} else if (trace->duration_filter)
		goto out;

	if (sample->callchain) {
		callchain_ret = trace__resolve_callchain(trace, evsel, sample, &callchain_cursor);
		if (callchain_ret == 0) {
			if (callchain_cursor.nr < trace->min_stack)
				goto out;
			callchain_ret = 1;
		}
	}

	if (trace->summary_only)
		goto out;

@@ -1982,7 +1992,10 @@ signed_print:

	fputc('\n', trace->output);

	trace__fprintf_callchain(trace, evsel, sample);
	if (callchain_ret > 0)
		trace__fprintf_callchain(trace, sample);
	else if (callchain_ret < 0)
		pr_err("Problem processing %s callchain, skipping...\n", perf_evsel__name(evsel));
out:
	ttrace->entry_pending = false;
	err = 0;
@@ -2131,7 +2144,10 @@ static int trace__event_handler(struct trace *trace, struct perf_evsel *evsel,

	fprintf(trace->output, ")\n");

	trace__fprintf_callchain(trace, evsel, sample);
	if (sample->callchain) {
		if (trace__resolve_callchain(trace, evsel, sample, &callchain_cursor) == 0)
			trace__fprintf_callchain(trace, sample);
	}

	return 0;
}
@@ -3082,6 +3098,9 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused)
		     &record_parse_callchain_opt),
	OPT_BOOLEAN(0, "kernel-syscall-graph", &trace.kernel_syscallchains,
		    "Show the kernel callchains on the syscall exit path"),
	OPT_UINTEGER(0, "min-stack", &trace.min_stack,
		     "Set the minimum stack depth when parsing the callchain, "
		     "anything below the specified depth will be ignored."),
	OPT_UINTEGER(0, "max-stack", &trace.max_stack,
		     "Set the maximum stack depth when parsing the callchain, "
		     "anything beyond the specified depth will be ignored. "