ftrace学习-以gpio为例

如果需要跟踪函数,需要打开内核 DYNAMIC_FTRACE 选项,下面以gpio为例具体分析:
详细内核配置:

// 内核配置
linux-4.xx$ cat .config | grep FTRACE
# CONFIG_PSTORE_FTRACE is not set
CONFIG_HAVE_DYNAMIC_FTRACE=y
CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
CONFIG_FTRACE=y
CONFIG_FTRACE_SYSCALLS=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FTRACE_MCOUNT_RECORD=y
# CONFIG_FTRACE_STARTUP_TEST is not set

准备步骤:

// 系统启动之后进入sys文件系统
/sys/kernel/debug# cat gpio 
gpiochip1: GPIOs 480-495, parent: platform/28005000.gpio, 28005000.gpio:
 gpio-480 (                    |sysfs               ) in  lo    

gpiochip0: GPIOs 496-511, parent: platform/28004000.gpio, 28004000.gpio:

// 查看支持的gpio跟踪函数
/sys/kernel/debug/tracing# cat set_graph_function
#### all functions enabled ####

/sys/kernel/debug/tracing# echo gpiod_* > set_graph_function 
/sys/kernel/debug/tracing# cat set_graph_function 
gpiod_get_raw_array_value
gpiod_set_array_value_cansleep
gpiod_configure_flags
gpiod_to_chip
gpiod_is_active_low
gpiod_request_commit
gpiod_get_raw_value_cansleep
gpiod_add_lookup_tables
gpiod_set_raw_value_cansleep
gpiod_get_raw_value
gpiod_direction_output_raw_commit
gpiod_add_hogs
gpiod_get_value_cansleep
gpiod_get_raw_array_value_cansleep
gpiod_free
gpiod_set_raw_array_value
gpiod_set_value
gpiod_get_array_value_complex
gpiod_set_value_cansleep
gpiod_get
gpiod_to_irq
gpiod_get_index_optional
gpiod_set_raw_array_value_cansleep
gpiod_get_optional
gpiod_set_raw_value
gpiod_add_lookup_table
gpiod_get_direction
gpiod_unexport
gpiod_export_link
gpiod_set_array_value_complex
gpiod_get_array
gpiod_cansleep
gpiod_hog
gpiod_set_debounce
gpiod_set_value_nocheck
gpiod_remove_lookup_table
gpiod_get_array_value
gpiod_put
gpiod_get_array_optional
gpiod_get_raw_value_commit
gpiod_free_commit
gpiod_find_lookup_table
gpiod_get_index
gpiod_direction_input
gpiod_export
gpiod_direction_output
gpiod_set_transitory
gpiod_direction_output_raw
gpiod_get_from_of_node
gpiod_set_array_value
gpiod_request
gpiod_get_array_value_cansleep
gpiod_set_consumer_name
gpiod_count
gpiod_get_value
gpiod_set_raw_value_commit
gpiod_put_array


/sys/kernel/debug/tracing# cat current_tracer 
nop

/sys/kernel/debug/tracing# cat available_tracers 
hwlat function_graph wakeup_dl wakeup_rt wakeup function nop

/sys/kernel/debug/tracing# echo function_graph > current_tracer 
/sys/kernel/debug/tracing# cat current_tracer 
function_graph

详细操作步骤:

// 重新打开一个新的窗口用来操作GPIO
// 实验1:导出GPIO管脚设备
/sys/class/gpio# echo 480 > export

// 重新回到原来的窗口查看结果:
/sys/kernel/debug/tracing# cat trace_pipe
 1)               |  gpiod_request() {
 1)   ==========> |
 1)               |    gic_handle_irq() {
 1)               |      handle_IPI() {
 1)               |        irq_enter() {
 1)               |          rcu_irq_enter() {
 1)               |            rcu_nmi_enter() {
 1)   1.625 us    |              rcu_dynticks_curr_cpu_in_eqs();
 1)   5.625 us    |            }
 1)   9.250 us    |          }
 1)   1.875 us    |          irqtime_account_irq();
 1) + 16.042 us   |        }
 1)               |        __wake_up() {
 1)               |          __wake_up_common_lock() {
 1)   1.833 us    |            _raw_spin_lock_irqsave();
 1)               |            __wake_up_common() {
 1)               |              autoremove_wake_function() {
 1)               |                default_wake_function() {
 1)               |                  try_to_wake_up() {
 1)   1.708 us    |                    _raw_spin_lock_irqsave();
 1)               |                    select_task_rq_fair() {
 1)   1.563 us    |                      __rcu_read_lock();
 1)   1.604 us    |                      available_idle_cpu();
 1)   2.667 us    |                      update_cfs_rq_h_load();
 1)               |                      select_idle_sibling() {
 1)   1.563 us    |                        available_idle_cpu();
 1)   4.812 us    |                      }
 1)   1.542 us    |                      __rcu_read_unlock();
 1) + 22.917 us   |                    }
 1)               |                    smp_send_reschedule() {
 1)               |                      smp_cross_call() {
 1)   1.854 us    |                        gic_raise_softirq();
 1)   5.021 us    |                      }
 1)   8.437 us    |                    }
 1)   1.667 us    |                    _raw_spin_unlock_irqrestore();
 1) + 43.937 us   |                  }
 1) + 47.146 us   |                }
 1) + 50.270 us   |              }
 1) + 53.834 us   |            }
 1)   1.667 us    |            _raw_spin_unlock_irqrestore();
 1) + 63.958 us   |          }
 1) + 67.250 us   |        }
 1)               |        irq_exit() {
 1)               |          irqtime_account_irq() {
 1)   1.542 us    |            irqtime_account_delta.isra.0();
 1)   4.958 us    |          }
 1)   1.812 us    |          idle_cpu();
 1)               |          rcu_irq_exit() {
 1)               |            rcu_nmi_exit() {
 1)   1.563 us    |              rcu_dynticks_curr_cpu_in_eqs();
 1)   4.750 us    |            }
 1)   8.062 us    |          }
 1) + 21.459 us   |        }
 1) ! 112.708 us  |      }
 1) ! 116.771 us  |    }
 1)   <========== |
 1)   1.875 us    |    validate_desc();
 1)   2.230 us    |    try_module_get();
 1)               |    gpiod_request_commit() {
 1)   1.709 us    |      kstrdup_const();
 1)   1.771 us    |      _raw_spin_lock_irqsave();
 1)   1.875 us    |      _raw_spin_unlock_irqrestore();
 1)               |      gpiod_get_direction() {
 1)               |        XXX_gpio_get_direction [gpio_XXX]() {	// XXX代表具体厂家的芯片驱动前缀,下同
 1)   1.542 us    |          gpiochip_get_data();
 1)   5.291 us    |        }
 1)   8.937 us    |      }
 1)   1.771 us    |      _raw_spin_lock_irqsave();
 1)   1.792 us    |      _raw_spin_unlock_irqrestore();
 1) + 30.354 us   |    }
 1)   1.979 us    |    get_device();
 1) ! 172.500 us  |  }
 1)               |  gpiod_set_transitory() {
 1)   2.292 us    |    validate_desc();
 1)   7.667 us    |  }
 1)               |  gpiod_export() {
 1)   1.770 us    |    mutex_lock();
 1)   2.062 us    |    _raw_spin_lock_irqsave();
 1)   1.875 us    |    _raw_spin_unlock_irqrestore();
 1)               |    kmem_cache_alloc_trace() {
 1)   1.667 us    |      should_failslab();
 1)   2.188 us    |      memcg_kmem_put_cache();
 1) + 10.750 us   |    }
 1)   1.666 us    |    __mutex_init();
 1)   3.542 us    |    desc_to_gpio();
 1)               |    device_create_with_groups() {
 1)               |      device_create_groups_vargs() {
 1)               |        kmem_cache_alloc_trace() {
 1)   1.563 us    |          should_failslab();
 1)   1.792 us    |          memcg_kmem_put_cache();
 1)   9.791 us    |        }
 1)               |        device_initialize() {
 1)   1.562 us    |          __mutex_init();
 1)               |          device_pm_sleep_init() {
 1)   1.541 us    |            __init_waitqueue_head();
 1)               |            complete_all() {
 1)   2.042 us    |              _raw_spin_lock_irqsave();
 1)               |              __wake_up_locked() {
 1)   1.646 us    |                __wake_up_common();
 1)   5.000 us    |              }
 1)   1.792 us    |              _raw_spin_unlock_irqrestore();
 1) + 15.937 us   |            }
 1) + 23.291 us   |          }
 1)               |          pm_runtime_init() {
 1)   1.563 us    |            init_timer_key();
 1)   1.542 us    |            __init_waitqueue_head();
 1)   8.021 us    |          }
 1) + 39.813 us   |        }
 1)               |        __kmalloc_track_caller() {
 1)   1.583 us    |          kmalloc_slab();
 1)   1.562 us    |          should_failslab();
 1)   1.562 us    |          memcg_kmem_put_cache();
 1) + 12.395 us   |        }
 1)               |        kfree_const() {
 1)   1.563 us    |          kfree();
 1)   4.792 us    |        }
 1)               |        device_add() {
 1)               |          kmem_cache_alloc_trace() {
 1)   1.875 us    |            should_failslab();
 1)               |            __slab_alloc.isra.24() {
 1)   2.792 us    |              ___slab_alloc();
 1)   6.417 us    |            }
 1)   2.041 us    |            memcg_kmem_put_cache();
 1) + 18.188 us   |          }
 1)               |          get_device_parent.isra.16() {
 1)   1.646 us    |            mutex_lock();
 1)   1.625 us    |            _raw_spin_lock();
 1)   2.187 us    |            _raw_spin_unlock();
 1)               |            kmem_cache_alloc_trace() {
 1)   1.562 us    |              should_failslab();
 1)   1.584 us    |              memcg_kmem_put_cache();
 1)   8.333 us    |            }
 1)   1.813 us    |            kstrdup_const();
 1)               |            kfree_const() {
 1)   1.563 us    |              kfree();
 1)   4.792 us    |            }
 1)   1.583 us    |            _raw_spin_lock();
 1)   1.938 us    |            _raw_spin_unlock();
 1)               |            sysfs_create_dir_ns() {
 1)               |              kernfs_create_dir_ns() {
 1)               |                kernfs_new_node() {
 1)               |                  __kernfs_new_node() {
 1)   1.896 us    |                    kstrdup_const();
 1)               |                    kmem_cache_alloc() {
 1)   1.542 us    |                      should_failslab();
 1)   1.646 us    |                      memcg_kmem_put_cache();
 1)   8.855 us    |                    }
 1)   1.958 us    |                    _raw_spin_lock();
 1)   1.687 us    |                    _raw_spin_unlock();
 1) + 26.729 us   |                  }
 1)   1.625 us    |                  kernfs_get();
 1) + 33.479 us   |                }
 1)               |                kernfs_add_one() {
 1)   1.562 us    |                  mutex_lock();
 1)   1.979 us    |                  kernfs_name_hash();
 1)   2.521 us    |                  kernfs_link_sibling();
 1)   1.605 us    |                  mutex_unlock();
 1)               |                  kernfs_activate() {
 1)   1.583 us    |                    mutex_lock();
 1)   1.729 us    |                    kernfs_next_descendant_post();
 1)   1.625 us    |                    kernfs_next_descendant_post();
 1)   3.333 us    |                    mutex_unlock();
 1) + 17.938 us   |                  }
 1) + 36.979 us   |                }
 1) + 75.687 us   |              }
 1) + 78.980 us   |            }
 1)   1.562 us    |            kernfs_get();
 1)   1.542 us    |            class_dir_child_ns_type();
 1)   2.167 us    |            mutex_unlock();
 1) ! 134.521 us  |          }
 1)   1.833 us    |          _raw_spin_lock();
 1)   1.750 us    |          _raw_spin_unlock();
 1)   1.604 us    |          class_dir_child_ns_type();
 1)               |          sysfs_create_dir_ns() {
 1)   2.084 us    |            device_get_ownership();
 1)               |            kernfs_create_dir_ns() {
 1)               |              kernfs_new_node() {
 1)               |                __kernfs_new_node() {
 1)               |                  kstrdup_const() {
 1)               |                    kstrdup() {
 1)               |                      __kmalloc_track_caller() {
 1)   1.562 us    |                        kmalloc_slab();
 1)   2.417 us    |                        should_failslab();
 1)   2.125 us    |                        memcg_kmem_put_cache();
 1) + 13.333 us   |                      }
 1) + 16.562 us   |                    }
 1) + 19.750 us   |                  }
 1)               |                  kmem_cache_alloc() {
 1)   1.541 us    |                    should_failslab();
 1)   1.750 us    |                    memcg_kmem_put_cache();
 1)   8.417 us    |                  }
 1)   2.396 us    |                  _raw_spin_lock();
 1)   1.896 us    |                  _raw_spin_unlock();
 1) + 41.708 us   |                }
 1)   1.625 us    |                kernfs_get();
 1) + 48.230 us   |              }
 1)               |              kernfs_add_one() {
 1)   1.562 us    |                mutex_lock();
 1)   2.437 us    |                kernfs_name_hash();
 1)   1.708 us    |                kernfs_link_sibling();
 1)   1.625 us    |                mutex_unlock();
 1)               |                kernfs_activate() {
 1)   1.563 us    |                  mutex_lock();
 1)   1.937 us    |                  kernfs_next_descendant_post();
 1)   1.646 us    |                  kernfs_next_descendant_post();
 1)   1.562 us    |                  mutex_unlock();
 1) + 15.271 us   |                }
 1) + 33.270 us   |              }
 1) + 88.125 us   |            }
 1) + 96.167 us   |          }
 1)   1.562 us    |          kernfs_get();
 1)               |          device_create_file() {
 1)               |            sysfs_create_file_ns() {
 1)   1.750 us    |              device_get_ownership();
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.625 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   2.166 us    |                        should_failslab();
 1)   1.791 us    |                        memcg_kmem_put_cache();
 1)   9.521 us    |                      }
 1)   1.605 us    |                      _raw_spin_lock();
 1)   1.750 us    |                      _raw_spin_unlock();
 1) + 23.083 us   |                    }
 1)   1.604 us    |                    kernfs_get();
 1) + 30.604 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.812 us    |                    mutex_lock();
 1)   1.708 us    |                    kernfs_name_hash();
 1)   1.562 us    |                    kernfs_link_sibling();
 1)   1.625 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.812 us    |                      mutex_lock();
 1)   1.521 us    |                      kernfs_next_descendant_post();
 1)   1.542 us    |                      kernfs_next_descendant_post();
 1)   2.104 us    |                      mutex_unlock();
 1) + 17.416 us   |                    }
 1) + 35.000 us   |                  }
 1) + 70.687 us   |                }
 1) + 74.604 us   |              }
 1) + 81.792 us   |            }
 1) + 85.062 us   |          }
 1)               |          sysfs_create_link() {
 1)               |            sysfs_do_create_link_sd.isra.0() {
 1)   1.750 us    |              _raw_spin_lock();
 1)   1.750 us    |              kernfs_get();
 1)   1.625 us    |              _raw_spin_unlock();
 1)               |              kernfs_create_link() {
 1)               |                kernfs_new_node() {
 1)               |                  __kernfs_new_node() {
 1)   1.604 us    |                    kstrdup_const();
 1)               |                    kmem_cache_alloc() {
 1)   1.583 us    |                      should_failslab();
 1)   2.500 us    |                      memcg_kmem_put_cache();
 1)   9.438 us    |                    }
 1)   1.667 us    |                    _raw_spin_lock();
 1)   1.813 us    |                    _raw_spin_unlock();
 1) + 23.896 us   |                  }
 1)   1.979 us    |                  kernfs_get();
 1) + 31.334 us   |                }
 1)   1.625 us    |                kernfs_get();
 1)               |                kernfs_add_one() {
 1)   1.646 us    |                  mutex_lock();
 1)   2.166 us    |                  kernfs_name_hash();
 1)   1.792 us    |                  kernfs_link_sibling();
 1)   1.583 us    |                  mutex_unlock();
 1)               |                  kernfs_activate() {
 1)   1.979 us    |                    mutex_lock();
 1)   1.583 us    |                    kernfs_next_descendant_post();
 1)   1.625 us    |                    kernfs_next_descendant_post();
 1)   1.667 us    |                    mutex_unlock();
 1) + 15.375 us   |                  }
 1) + 34.167 us   |                }
 1) + 74.229 us   |              }
 1)   1.583 us    |              kernfs_put();
 1) + 92.041 us   |            }
 1) + 95.895 us   |          }
 1)               |          sysfs_create_link() {
 1)               |            sysfs_do_create_link_sd.isra.0() {
 1)   1.771 us    |              _raw_spin_lock();
 1)   1.771 us    |              kernfs_get();
 1)   1.958 us    |              _raw_spin_unlock();
 1)               |              kernfs_create_link() {
 1)               |                kernfs_new_node() {
 1)               |                  __kernfs_new_node() {
 1)   1.583 us    |                    kstrdup_const();
 1)               |                    kmem_cache_alloc() {
 1)   1.917 us    |                      should_failslab();
 1)   1.625 us    |                      memcg_kmem_put_cache();
 1)   9.583 us    |                    }
 1)   1.646 us    |                    _raw_spin_lock();
 1)               |                    kmem_cache_alloc() {
 1)   1.625 us    |                      should_failslab();
 1)   1.625 us    |                      memcg_kmem_put_cache();
 1)   8.896 us    |                    }
 1)   2.583 us    |                    _raw_spin_unlock();
 1) + 36.458 us   |                  }
 1)   ==========> |
 1)               |                  gic_handle_irq() {
 1)               |                    __handle_domain_irq() {
 1)               |                      irq_enter() {
 1)               |                        rcu_irq_enter() {
 1)               |                          rcu_nmi_enter() {
 1)   1.625 us    |                            rcu_dynticks_curr_cpu_in_eqs();
 1)   4.791 us    |                          }
 1)   7.980 us    |                        }
 1)   1.770 us    |                        irqtime_account_irq();
 1) + 14.750 us   |                      }
 1)               |                      irq_find_mapping() {
 1)   1.625 us    |                        __rcu_read_lock();
 1)   1.604 us    |                        __rcu_read_unlock();
 1)   8.708 us    |                      }
 1)               |                      generic_handle_irq() {
 1)               |                        handle_percpu_devid_irq() {
 1)               |                          arch_timer_handler_phys() {
 1)               |                            hrtimer_interrupt() {
 1)   1.687 us    |                              _raw_spin_lock_irqsave();
 1)               |                              ktime_get_update_offsets_now() {
 1)   1.646 us    |                                arch_counter_read();
 1)   4.979 us    |                              }
 1)               |                              __hrtimer_run_queues() {
 1)   1.917 us    |                                __remove_hrtimer();
 1)   1.625 us    |                                _raw_spin_unlock_irqrestore();
 1)               |                                tick_sched_timer() {
 1)               |                                  ktime_get() {
 1)   1.667 us    |                                    arch_counter_read();
 1)   4.833 us    |                                  }
 1)   1.646 us    |                                  tick_sched_do_timer();
 1)               |                                  tick_sched_handle.isra.5() {
 1)               |                                    update_process_times() {
 1)               |                                      account_process_tick() {
 1)               |                                        irqtime_account_process_tick.isra.1() {
 1)               |                                          account_system_index_time() {
 1)               |                                            cpuacct_account_field() {
 1)   1.521 us    |                                              __rcu_read_lock();
 1)   1.541 us    |                                              __rcu_read_unlock();
 1)   7.854 us    |                                            }
 1)   1.521 us    |                                            __rcu_read_lock();
 1)               |                                            __cgroup_account_cputime_field() {
 1)   1.542 us    |                                              cgroup_base_stat_cputime_account_begin();
 1)               |                                              cgroup_base_stat_cputime_account_end.isra.0() {
 1)   1.605 us    |                                                cgroup_rstat_updated();
 1)   4.791 us    |                                              }
 1) + 11.188 us   |                                            }
 1)   1.583 us    |                                            __rcu_read_unlock();
 1)               |                                            acct_account_cputime() {
 1)   1.583 us    |                                              __acct_update_integrals();
 1)   4.708 us    |                                            }
 1) + 37.000 us   |                                          }
 1) + 40.313 us   |                                        }
 1) + 43.542 us   |                                      }
 1)               |                                      run_local_timers() {
 1)   1.563 us    |                                        hrtimer_run_queues();
 1)               |                                        raise_softirq() {
 1)   1.521 us    |                                          __raise_softirq_irqoff();
 1)   4.937 us    |                                        }
 1) + 11.229 us   |                                      }
 1)               |                                      rcu_check_callbacks() {
 1)   1.583 us    |                                        rcu_is_cpu_rrupt_from_idle();
 1)   1.521 us    |                                        rcu_bh_qs();
 1)   1.687 us    |                                        rcu_preempt_qs();
 1)               |                                        invoke_rcu_core() {
 1)               |                                          raise_softirq() {
 1)   1.542 us    |                                            __raise_softirq_irqoff();
 1)   4.896 us    |                                          }
 1)   8.041 us    |                                        }
 1) + 21.437 us   |                                      }
 1)               |                                      scheduler_tick() {
 1)   1.542 us    |                                        _raw_spin_lock();
 1)               |                                        update_rq_clock.part.17() {
 1)   1.771 us    |                                          update_irq_load_avg();
 1)   5.208 us    |                                        }
 1)               |                                        task_tick_fair() {
 1)               |                                          update_curr() {
 1)   1.937 us    |                                            update_min_vruntime();
 1)               |                                            cpuacct_charge() {
 1)   1.979 us    |                                              __rcu_read_lock();
 1)   1.709 us    |                                              __rcu_read_unlock();
 1)   8.958 us    |                                            }
 1)   1.542 us    |                                            __rcu_read_lock();
 1)               |                                            __cgroup_account_cputime() {
 1)   1.542 us    |                                              cgroup_base_stat_cputime_account_begin();
 1)               |                                              cgroup_base_stat_cputime_account_end.isra.0() {
 1)   1.750 us    |                                                cgroup_rstat_updated();
 1)   5.375 us    |                                              }
 1) + 12.395 us   |                                            }
 1)   1.541 us    |                                            __rcu_read_unlock();
 1) + 36.854 us   |                                          }
 1)               |                                          __update_load_avg_se() {
 1)   2.083 us    |                                            __accumulate_pelt_segments();
 1)   5.625 us    |                                          }
 1)               |                                          __update_load_avg_cfs_rq() {
 1)   1.562 us    |                                            __accumulate_pelt_segments();
 1)   5.208 us    |                                          }
 1)   1.542 us    |                                          update_cfs_group();
 1)   1.541 us    |                                          hrtimer_active();
 1)               |                                          update_curr() {
 1)   1.563 us    |                                            update_min_vruntime();
 1)   5.062 us    |                                          }
 1)               |                                          __update_load_avg_se() {
 1)   1.583 us    |                                            __accumulate_pelt_segments();
 1)   5.792 us    |                                          }
 1)               |                                          __update_load_avg_cfs_rq() {
 1)   1.917 us    |                                            __accumulate_pelt_segments();
 1)   5.605 us    |                                          }
 1)               |                                          dbs_update_util_handler() {
 1)   1.625 us    |                                            cpufreq_this_cpu_can_update();
 1)   4.875 us    |                                          }
 1)               |                                          update_cfs_group() {
 1)               |                                            reweight_entity() {
 1)   1.812 us    |                                              update_curr();
 1)   1.625 us    |                                              account_entity_dequeue();
 1)   1.771 us    |                                              account_entity_enqueue();
 1) + 12.500 us   |                                            }
 1) + 16.979 us   |                                          }
 1)   1.604 us    |                                          hrtimer_active();
 1) ! 114.458 us  |                                        }
 1)               |                                        cpu_load_update_active() {
 1)   1.562 us    |                                          tick_nohz_tick_stopped();
 1)   2.583 us    |                                          cpu_load_update();
 1)   9.208 us    |                                        }
 1)   1.521 us    |                                        calc_global_load_tick();
 1)   1.563 us    |                                        _raw_spin_unlock();
 1)               |                                        trigger_load_balance() {
 1)               |                                          raise_softirq() {
 1)   1.709 us    |                                            __raise_softirq_irqoff();
 1)   6.125 us    |                                          }
 1)               |                                          nohz_balance_exit_idle() {
 1)   1.562 us    |                                            __rcu_read_lock();
 1)   1.604 us    |                                            __rcu_read_unlock();
 1)   8.625 us    |                                          }
 1)   1.542 us    |                                          __rcu_read_lock();
 1)   1.584 us    |                                          __rcu_read_unlock();
 1) + 27.521 us   |                                        }
 1) ! 175.000 us  |                                      }
 1)   2.021 us    |                                      run_posix_cpu_timers();
 1) ! 263.562 us  |                                    }
 1)   1.563 us    |                                    profile_tick();
 1) ! 269.958 us  |                                  }
 1)   1.604 us    |                                  hrtimer_forward();
 1) ! 286.145 us  |                                }
 1)   1.937 us    |                                _raw_spin_lock_irq();
 1)   1.813 us    |                                enqueue_hrtimer();
 1) ! 304.521 us  |                              }
 1)               |                              __hrtimer_get_next_event() {
 1)   1.687 us    |                                __hrtimer_next_event_base();
 1)   1.562 us    |                                __hrtimer_next_event_base();
 1)   9.270 us    |                              }
 1)   1.645 us    |                              _raw_spin_unlock_irqrestore();
 1)               |                              tick_program_event() {
 1)               |                                clockevents_program_event() {
 1)               |                                  ktime_get() {
 1)   1.667 us    |                                    arch_counter_read();
 1)   4.812 us    |                                  }
 1)   2.083 us    |                                  arch_timer_set_next_event_phys();
 1) + 12.187 us   |                                }
 1) + 15.458 us   |                              }
 1) ! 351.521 us  |                            }
 1) ! 355.209 us  |                          }
 1)   1.645 us    |                          gic_eoimode1_eoi_irq();
 1) ! 361.771 us  |                        }
 1) ! 365.250 us  |                      }
 1)               |                      irq_exit() {
 1)               |                        irqtime_account_irq() {
 1)   1.855 us    |                          irqtime_account_delta.isra.0();
 1)   5.687 us    |                        }
 1)   1.708 us    |                        ksoftirqd_running();
 1)               |                        __do_softirq() {
 1)   1.813 us    |                          irqtime_account_irq();
 1)               |                          run_timer_softirq() {
 1)   1.959 us    |                            _raw_spin_lock_irq();
 1)               |                            collect_expired_timers() {
 1)   2.125 us    |                              __next_timer_interrupt();
 1)   5.417 us    |                            }
 1)   2.250 us    |                            _raw_spin_unlock_irq();
 1)   1.854 us    |                            _raw_spin_lock_irq();
 1)               |                            collect_expired_timers() {
 1)   2.229 us    |                              __next_timer_interrupt();
 1)   5.813 us    |                            }
 1)   2.021 us    |                            _raw_spin_unlock_irq();
 1) + 33.812 us   |                          }
 1)               |                          run_rebalance_domains() {
 1)               |                            update_blocked_averages() {
 1)   1.750 us    |                              _raw_spin_lock_irqsave();
 1)               |                              update_rq_clock() {
 1)               |                                update_rq_clock.part.17() {
 1)   1.646 us    |                                  update_irq_load_avg();
 1)   5.375 us    |                                }
 1)   9.812 us    |                              }
 1)   1.688 us    |                              __update_load_avg_cfs_rq();
 1)   2.021 us    |                              __update_load_avg_se();
 1)   1.792 us    |                              __update_load_avg_cfs_rq();
 1)   1.583 us    |                              __update_load_avg_cfs_rq();
 1)               |                              update_rt_rq_load_avg() {
 1)   1.625 us    |                                __accumulate_pelt_segments();
 1)   5.062 us    |                              }
 1)               |                              update_dl_rq_load_avg() {
 1)   1.583 us    |                                __accumulate_pelt_segments();
 1)   6.062 us    |                              }
 1)   1.875 us    |                              update_irq_load_avg();
 1)   1.667 us    |                              _raw_spin_unlock_irqrestore();
 1) + 53.208 us   |                            }
 1)               |                            rebalance_domains() {
 1)   1.583 us    |                              __rcu_read_lock();
 1)   1.563 us    |                              __msecs_to_jiffies();
 1)               |                              load_balance() {
 1)   1.604 us    |                                idle_cpu();
 1)   1.916 us    |                                group_balance_cpu();
 1)               |                                find_busiest_group() {
 1)               |                                  update_group_capacity() {
 1)   1.604 us    |                                    __msecs_to_jiffies();
 1)   4.896 us    |                                  }
 1)   2.145 us    |                                  idle_cpu();
 1) + 14.208 us   |                                }
 1) + 26.375 us   |                              }
 1)   1.583 us    |                              __msecs_to_jiffies();
 1)   1.646 us    |                              __msecs_to_jiffies();
 1)   2.563 us    |                              __rcu_read_unlock();
 1) + 47.583 us   |                            }
 1) ! 106.250 us  |                          }
 1)               |                          rcu_process_callbacks() {
 1)               |                            note_gp_changes() {
 1)   1.667 us    |                              _raw_spin_trylock();
 1)               |                              __note_gp_changes() {
 1)               |                                rcu_advance_cbs() {
 1)   1.583 us    |                                  rcu_segcblist_pend_cbs();
 1)   4.834 us    |                                }
 1)   9.229 us    |                              }
 1)   2.000 us    |                              _raw_spin_unlock_irqrestore();
 1) + 20.375 us   |                            }
 1)   1.563 us    |                            rcu_jiffies_till_stall_check();
 1)   1.625 us    |                            rcu_segcblist_ready_cbs();
 1)   1.792 us    |                            note_gp_changes();
 1)   1.562 us    |                            rcu_jiffies_till_stall_check();
 1)   1.958 us    |                            rcu_segcblist_ready_cbs();
 1)   2.083 us    |                            note_gp_changes();
 1)   1.583 us    |                            rcu_jiffies_till_stall_check();
 1)   1.562 us    |                            rcu_segcblist_ready_cbs();
 1) + 52.604 us   |                          }
 1)   1.583 us    |                          rcu_bh_qs();
 1)               |                          irqtime_account_irq() {
 1)   1.979 us    |                            irqtime_account_delta.isra.0();
 1)   5.417 us    |                          }
 1) ! 214.584 us  |                        }
 1)   1.813 us    |                        idle_cpu();
 1)               |                        rcu_irq_exit() {
 1)               |                          rcu_nmi_exit() {
 1)   1.583 us    |                            rcu_dynticks_curr_cpu_in_eqs();
 1)   4.916 us    |                          }
 1)   8.229 us    |                        }
 1) ! 242.708 us  |                      }
 1) ! 640.625 us  |                    }
 1) ! 645.854 us  |                  }
 1)   <========== |
 1)   1.813 us    |                  kernfs_get();
 1) ! 692.104 us  |                }
 1)   1.605 us    |                kernfs_get();
 1)               |                kernfs_add_one() {
 1)   1.771 us    |                  mutex_lock();
 1)   2.146 us    |                  kernfs_name_hash();
 1)   1.958 us    |                  kernfs_link_sibling();
 1)   1.604 us    |                  mutex_unlock();
 1)               |                  kernfs_activate() {
 1)   2.104 us    |                    mutex_lock();
 1)   1.854 us    |                    kernfs_next_descendant_post();
 1)   1.584 us    |                    kernfs_next_descendant_post();
 1)   1.583 us    |                    mutex_unlock();
 1) + 15.875 us   |                  }
 1) + 34.292 us   |                }
 1) ! 734.750 us  |              }
 1)   1.625 us    |              kernfs_put();
 1) ! 753.646 us  |            }
 1) ! 757.292 us  |          }
 1)               |          sysfs_create_link() {
 1)               |            sysfs_do_create_link_sd.isra.0() {
 1)   1.604 us    |              _raw_spin_lock();
 1)   1.666 us    |              kernfs_get();
 1)   1.625 us    |              _raw_spin_unlock();
 1)               |              kernfs_create_link() {
 1)               |                kernfs_new_node() {
 1)               |                  __kernfs_new_node() {
 1)               |                    kstrdup_const() {
 1)               |                      kstrdup() {
 1)               |                        __kmalloc_track_caller() {
 1)   1.604 us    |                          kmalloc_slab();
 1)   1.604 us    |                          should_failslab();
 1)   1.646 us    |                          memcg_kmem_put_cache();
 1) + 11.958 us   |                        }
 1) + 16.584 us   |                      }
 1) + 20.125 us   |                    }
 1)               |                    kmem_cache_alloc() {
 1)   1.562 us    |                      should_failslab();
 1)   1.896 us    |                      memcg_kmem_put_cache();
 1)   9.375 us    |                    }
 1)   1.625 us    |                    _raw_spin_lock();
 1)   1.709 us    |                    _raw_spin_unlock();
 1) + 42.709 us   |                  }
 1)   2.042 us    |                  kernfs_get();
 1) + 50.521 us   |                }
 1)   1.605 us    |                kernfs_get();
 1)               |                kernfs_add_one() {
 1)   1.605 us    |                  mutex_lock();
 1)   2.021 us    |                  kernfs_name_hash();
 1)   2.167 us    |                  kernfs_link_sibling();
 1)   1.604 us    |                  mutex_unlock();
 1)               |                  kernfs_activate() {
 1)   1.709 us    |                    mutex_lock();
 1)   1.896 us    |                    kernfs_next_descendant_post();
 1)   1.584 us    |                    kernfs_next_descendant_post();
 1)   1.605 us    |                    mutex_unlock();
 1) + 15.500 us   |                  }
 1) + 33.146 us   |                }
 1) + 91.917 us   |              }
 1)   1.979 us    |              kernfs_put();
 1) ! 110.854 us  |            }
 1) ! 114.979 us  |          }
 1)   1.605 us    |          sysfs_create_groups();
 1)               |          sysfs_create_groups() {
 1)               |            internal_create_group() {
 1)   1.584 us    |              device_get_ownership();
 1)   2.167 us    |              kernfs_get();
 1)   1.895 us    |              gpio_is_visible();
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.646 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.958 us    |                        should_failslab();
 1)   1.730 us    |                        memcg_kmem_put_cache();
 1) + 11.292 us   |                      }
 1)   1.708 us    |                      _raw_spin_lock();
 1)   1.687 us    |                      _raw_spin_unlock();
 1) + 26.584 us   |                    }
 1)   1.750 us    |                    kernfs_get();
 1) + 33.542 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.562 us    |                    mutex_lock();
 1)   2.646 us    |                    kernfs_name_hash();
 1)   1.854 us    |                    kernfs_link_sibling();
 1)   1.562 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.980 us    |                      mutex_lock();
 1)   1.563 us    |                      kernfs_next_descendant_post();
 1)   1.542 us    |                      kernfs_next_descendant_post();
 1)   1.563 us    |                      mutex_unlock();
 1) + 14.958 us   |                    }
 1) + 33.541 us   |                  }
 1) + 72.542 us   |                }
 1) + 76.167 us   |              }
 1)               |              gpio_is_visible() {
 1)               |                gpiod_to_irq() {
 1)               |                  gpiochip_to_irq() {
 1)   2.041 us    |                    gpiochip_irqchip_irq_valid();
 1)               |                    irq_create_mapping() {
 1)   1.708 us    |                      irq_find_mapping();
 1)               |                      of_node_to_nid() {
 1)   1.771 us    |                        of_node_get();
 1)               |                        of_property_read_variable_u32_array() {
 1)               |                          of_find_property_value_of_size() {
 1)               |                            of_find_property() {
 1)   1.667 us    |                              _raw_spin_lock_irqsave();
 1)   3.834 us    |                              __of_find_property();
 1)   1.979 us    |                              _raw_spin_unlock_irqrestore();
 1) + 14.771 us   |                            }
 1) + 18.396 us   |                          }
 1) + 22.145 us   |                        }
 1)               |                        of_get_next_parent() {
 1)   1.688 us    |                          _raw_spin_lock_irqsave();
 1)   1.750 us    |                          of_node_get();
 1)   1.875 us    |                          of_node_put();
 1)   1.667 us    |                          _raw_spin_unlock_irqrestore();
 1) + 15.875 us   |                        }
 1)               |                        of_property_read_variable_u32_array() {
 1)               |                          of_find_property_value_of_size() {
 1)               |                            of_find_property() {
 1)   1.687 us    |                              _raw_spin_lock_irqsave();
 1)   2.292 us    |                              __of_find_property();
 1)   1.584 us    |                              _raw_spin_unlock_irqrestore();
 1) + 13.021 us   |                            }
 1) + 16.208 us   |                          }
 1) + 19.480 us   |                        }
 1)               |                        of_get_next_parent() {
 1)   1.687 us    |                          _raw_spin_lock_irqsave();
 1)   1.563 us    |                          of_node_get();
 1)   1.562 us    |                          of_node_put();
 1)   1.646 us    |                          _raw_spin_unlock_irqrestore();
 1) + 14.667 us   |                        }
 1)               |                        of_property_read_variable_u32_array() {
 1)               |                          of_find_property_value_of_size() {
 1)               |                            of_find_property() {
 1)   1.667 us    |                              _raw_spin_lock_irqsave();
 1)   1.958 us    |                              __of_find_property();
 1)   1.625 us    |                              _raw_spin_unlock_irqrestore();
 1) + 11.980 us   |                            }
 1) + 15.666 us   |                          }
 1) + 19.334 us   |                        }
 1)               |                        of_get_next_parent() {
 1)   2.021 us    |                          _raw_spin_lock_irqsave();
 1)   1.542 us    |                          of_node_get();
 1)   1.584 us    |                          of_node_put();
 1)   1.666 us    |                          _raw_spin_unlock_irqrestore();
 1) + 15.167 us   |                        }
 1)   1.563 us    |                        of_node_put();
 1) ! 128.084 us  |                      }
 1)               |                      irq_domain_alloc_descs() {
 1)               |                        __irq_alloc_descs() {
 1)   1.688 us    |                          arch_dynirq_lower_bound();
 1)   1.666 us    |                          mutex_lock();
 1)               |                          alloc_desc() {
 1)               |                            kmem_cache_alloc_node_trace() {
 1)   2.500 us    |                              should_failslab();
 1)   1.562 us    |                              memcg_kmem_put_cache();
 1)   9.875 us    |                            }
 1)               |                            __alloc_percpu() {
 1)               |                              pcpu_alloc() {
 1)   2.521 us    |                                mutex_lock_killable();
 1)   1.875 us    |                                _raw_spin_lock_irqsave();
 1)               |                                pcpu_find_block_fit() {
 1)   1.834 us    |                                  pcpu_next_fit_region.constprop.15();
 1)   5.416 us    |                                }
 1)               |                                pcpu_alloc_area() {
 1)   1.542 us    |                                  pcpu_chunk_slot();
 1)               |                                  pcpu_block_update_hint_alloc() {
 1)               |                                    pcpu_block_refresh_hint() {
 1)   1.750 us    |                                      pcpu_next_unpop();
 1)   1.605 us    |                                      pcpu_block_update();
 1)   2.458 us    |                                      pcpu_next_unpop();
 1) + 12.979 us   |                                    }
 1) + 16.833 us   |                                  }
 1)               |                                  pcpu_chunk_relocate() {
 1)   1.563 us    |                                    pcpu_chunk_slot();
 1)   5.000 us    |                                  }
 1) + 31.959 us   |                                }
 1)   1.646 us    |                                _raw_spin_unlock_irqrestore();
 1)   2.792 us    |                                pcpu_next_unpop();
 1)   1.708 us    |                                mutex_unlock();
 1) + 65.208 us   |                              }
 1) + 68.500 us   |                            }
 1)   1.938 us    |                            __mutex_init();
 1) + 88.062 us   |                          }
 1)               |                          irq_sysfs_add() {
 1)               |                            __kmalloc_track_caller() {
 1)   2.084 us    |                              kmalloc_slab();
 1)   1.667 us    |                              should_failslab();
 1)   1.542 us    |                              memcg_kmem_put_cache();
 1) + 13.125 us   |                            }
 1)               |                            kfree_const() {
 1)   2.146 us    |                              kfree();
 1)   5.500 us    |                            }
 1)               |                            sysfs_create_dir_ns() {
 1)               |                              kernfs_create_dir_ns() {
 1)               |                                kernfs_new_node() {
 1)               |                                  __kernfs_new_node() {
 1)               |                                    kstrdup_const() {
 1)               |                                      kstrdup() {
 1)               |                                        __kmalloc_track_caller() {
 1)   1.938 us    |                                          kmalloc_slab();
 1)   1.541 us    |                                          should_failslab();
 1)   1.521 us    |                                          memcg_kmem_put_cache();
 1) + 12.208 us   |                                        }
 1) + 15.604 us   |                                      }
 1) + 18.750 us   |                                    }
 1)               |                                    kmem_cache_alloc() {
 1)   1.541 us    |                                      should_failslab();
 1)   1.813 us    |                                      memcg_kmem_put_cache();
 1)   9.250 us    |                                    }
 1)   1.563 us    |                                    _raw_spin_lock();
 1)   1.625 us    |                                    _raw_spin_unlock();
 1) + 40.667 us   |                                  }
 1)   1.979 us    |                                  kernfs_get();
 1) + 48.021 us   |                                }
 1)               |                                kernfs_add_one() {
 1)   1.604 us    |                                  mutex_lock();
 1)   1.729 us    |                                  kernfs_name_hash();
 1)   2.834 us    |                                  kernfs_link_sibling();
 1)   1.771 us    |                                  mutex_unlock();
 1)               |                                  kernfs_activate() {
 1)   1.584 us    |                                    mutex_lock();
 1)   1.604 us    |                                    kernfs_next_descendant_post();
 1)   2.708 us    |                                    kernfs_next_descendant_post();
 1)   1.563 us    |                                    mutex_unlock();
 1) + 16.000 us   |                                  }
 1) + 34.895 us   |                                }
 1) + 88.917 us   |                              }
 1) + 93.041 us   |                            }
 1)               |                            sysfs_create_file_ns() {
 1)               |                              sysfs_add_file_mode_ns() {
 1)               |                                __kernfs_create_file() {
 1)               |                                  kernfs_new_node() {
 1)               |                                    __kernfs_new_node() {
 1)   2.000 us    |                                      kstrdup_const();
 1)               |                                      kmem_cache_alloc() {
 1)   1.562 us    |                                        should_failslab();
 1)   1.583 us    |                                        memcg_kmem_put_cache();
 1)   8.791 us    |                                      }
 1)   1.688 us    |                                      _raw_spin_lock();
 1)   1.854 us    |                                      _raw_spin_unlock();
 1) + 24.250 us   |                                    }
 1)   1.980 us    |                                    kernfs_get();
 1) + 31.292 us   |                                  }
 1)               |                                  kernfs_add_one() {
 1)   1.583 us    |                                    mutex_lock();
 1)   1.854 us    |                                    kernfs_name_hash();
 1)   2.020 us    |                                    kernfs_link_sibling();
 1)   1.625 us    |                                    mutex_unlock();
 1)               |                                    kernfs_activate() {
 1)   1.771 us    |                                      mutex_lock();
 1)   1.605 us    |                                      kernfs_next_descendant_post();
 1)   1.875 us    |                                      kernfs_next_descendant_post();
 1)   1.604 us    |                                      mutex_unlock();
 1) + 15.896 us   |                                    } /* kernfs_activate */
 1) + 34.062 us   |                                  }
 1) + 70.230 us   |                                }
 1) + 73.667 us   |                              }
 1) + 76.979 us   |                            }
 1)               |                            sysfs_create_file_ns() {
 1)               |                              sysfs_add_file_mode_ns() {
 1)               |                                __kernfs_create_file() {
 1)               |                                  kernfs_new_node() {
 1)               |                                    __kernfs_new_node() {
 1)   1.604 us    |                                      kstrdup_const();
 1)               |                                      kmem_cache_alloc() {
 1)   1.563 us    |                                        should_failslab();
 1)   1.583 us    |                                        memcg_kmem_put_cache();
 1)   8.271 us    |                                      }
 1)   1.917 us    |                                      _raw_spin_lock();
 1)   1.875 us    |                                      _raw_spin_unlock();
 1) + 23.313 us   |                                    }
 1)   1.770 us    |                                    kernfs_get();
 1) + 30.209 us   |                                  }
 1)               |                                  kernfs_add_one() {
 1)   1.917 us    |                                    mutex_lock();
 1)   1.979 us    |                                    kernfs_name_hash();
 1)   1.646 us    |                                    kernfs_link_sibling();
 1)   1.605 us    |                                    mutex_unlock();
 1)               |                                    kernfs_activate() {
 1)   2.229 us    |                                      mutex_lock();
 1)   1.979 us    |                                      kernfs_next_descendant_post();
 1)   1.750 us    |                                      kernfs_next_descendant_post();
 1)   1.583 us    |                                      mutex_unlock();
 1) + 16.375 us   |                                    }
 1) + 33.855 us   |                                  }
 1) + 69.416 us   |                                }
 1) + 72.979 us   |                              }
 1) + 78.125 us   |                            }
 1)               |                            sysfs_create_file_ns() {
 1)               |                              sysfs_add_file_mode_ns() {
 1)               |                                __kernfs_create_file() {
 1)               |                                  kernfs_new_node() {
 1)               |                                    __kernfs_new_node() {
 1)   1.604 us    |                                      kstrdup_const();
 1)               |                                      kmem_cache_alloc() {
 1)   1.875 us    |                                        should_failslab();
 1)   1.604 us    |                                        memcg_kmem_put_cache();
 1)   8.916 us    |                                      }
 1)   1.625 us    |                                      _raw_spin_lock();
 1)   1.667 us    |                                      _raw_spin_unlock();
 1) + 23.437 us   |                                    }
 1)   2.709 us    |                                    kernfs_get();
 1) + 30.895 us   |                                  }
 1)               |                                  kernfs_add_one() {
 1)   1.604 us    |                                    mutex_lock();
 1)   1.875 us    |                                    kernfs_name_hash();
 1)   1.854 us    |                                    kernfs_link_sibling();
 1)   1.896 us    |                                    mutex_unlock();
 1)               |                                    kernfs_activate() {
 1)   1.583 us    |                                      mutex_lock();
 1)   1.563 us    |                                      kernfs_next_descendant_post();
 1)   1.583 us    |                                      kernfs_next_descendant_post();
 1)   1.562 us    |                                      mutex_unlock();
 1) + 15.958 us   |                                    }
 1) + 35.979 us   |                                  }
 1) + 72.312 us   |                                }
 1) + 75.605 us   |                              }
 1) + 78.979 us   |                            }
 1)               |                            sysfs_create_file_ns() {
 1)               |                              sysfs_add_file_mode_ns() {
 1)               |                                __kernfs_create_file() {
 1)               |                                  kernfs_new_node() {
 1)               |                                    __kernfs_new_node() {
 1)   2.021 us    |                                      kstrdup_const();
 1)               |                                      kmem_cache_alloc() {
 1)   1.854 us    |                                        should_failslab();
 1)   1.583 us    |                                        memcg_kmem_put_cache();
 1)   8.959 us    |                                      }
 1)   1.667 us    |                                      _raw_spin_lock();
 1)   2.083 us    |                                      _raw_spin_unlock();
 1) + 23.771 us   |                                    }
 1)   1.709 us    |                                    kernfs_get();
 1) + 30.312 us   |                                  }
 1)               |                                  kernfs_add_one() {
 1)   1.792 us    |                                    mutex_lock();
 1)   1.875 us    |                                    kernfs_name_hash();
 1)   1.729 us    |                                    kernfs_link_sibling();
 1)   1.729 us    |                                    mutex_unlock();
 1)               |                                    kernfs_activate() {
 1)   1.896 us    |                                      mutex_lock();
 1)   1.562 us    |                                      kernfs_next_descendant_post();
 1)   1.563 us    |                                      kernfs_next_descendant_post();
 1)   2.063 us    |                                      mutex_unlock();
 1) + 15.812 us   |                                    }
 1) + 33.646 us   |                                  }
 1) + 69.354 us   |                                }
 1) + 72.584 us   |                              }
 1) + 75.791 us   |                            }
 1)               |                            sysfs_create_file_ns() {
 1)               |                              sysfs_add_file_mode_ns() {
 1)               |                                __kernfs_create_file() {
 1)               |                                  kernfs_new_node() {
 1)               |                                    __kernfs_new_node() {
 1)   2.167 us    |                                      kstrdup_const();
 1)               |                                      kmem_cache_alloc() {
 1)   2.000 us    |                                        should_failslab();
 1)   1.563 us    |                                        memcg_kmem_put_cache();
 1)   9.250 us    |                                      }
 1)   1.584 us    |                                      _raw_spin_lock();
 1)   2.292 us    |                                      _raw_spin_unlock();
 1) + 24.605 us   |                                    }
 1)   1.750 us    |                                    kernfs_get();
 1) + 31.521 us   |                                  }
 1)               |                                  kernfs_add_one() {
 1)   1.562 us    |                                    mutex_lock();
 1)   2.041 us    |                                    kernfs_name_hash();
 1)   1.667 us    |                                    kernfs_link_sibling();
 1)   1.562 us    |                                    mutex_unlock();
 1)               |                                    kernfs_activate() {
 1)   2.459 us    |                                      mutex_lock();
 1)   1.563 us    |                                      kernfs_next_descendant_post();
 1)   1.563 us    |                                      kernfs_next_descendant_post();
 1)   1.584 us    |                                      mutex_unlock();
 1) + 15.729 us   |                                    }
 1) + 34.063 us   |                                  }
 1) + 70.354 us   |                                }
 1) + 73.792 us   |                              }
 1) + 77.291 us   |                            }
 1)               |                            sysfs_create_file_ns() {
 1)               |                              sysfs_add_file_mode_ns() {
 1)               |                                __kernfs_create_file() {
 1)               |                                  kernfs_new_node() {
 1)               |                                    __kernfs_new_node() {
 1)   2.042 us    |                                      kstrdup_const();
 1)               |                                      kmem_cache_alloc() {
 1)   1.563 us    |                                        should_failslab();
 1)   1.562 us    |                                        memcg_kmem_put_cache();
 1)   8.166 us    |                                      }
 1)   2.292 us    |                                      _raw_spin_lock();
 1)   1.937 us    |                                      _raw_spin_unlock();
 1) + 25.230 us   |                                    }
 1)   1.562 us    |                                    kernfs_get();
 1) + 31.563 us   |                                  }
 1)               |                                  kernfs_add_one() {
 1)   1.563 us    |                                    mutex_lock();
 1)   2.146 us    |                                    kernfs_name_hash();
 1)   1.708 us    |                                    kernfs_link_sibling();
 1)   1.542 us    |                                    mutex_unlock();
 1)               |                                    kernfs_activate() {
 1)   1.563 us    |                                      mutex_lock();
 1)   1.729 us    |                                      kernfs_next_descendant_post();
 1)   1.875 us    |                                      kernfs_next_descendant_post();
 1)   1.604 us    |                                      mutex_unlock();
 1) + 16.145 us   |                                    }
 1) + 33.875 us   |                                  }
 1) + 70.229 us   |                                }
 1) + 73.459 us   |                              }
 1) + 77.146 us   |                            }
 1)               |                            sysfs_create_file_ns() {
 1)               |                              sysfs_add_file_mode_ns() {
 1)               |                                __kernfs_create_file() {
 1)               |                                  kernfs_new_node() {
 1)               |                                    __kernfs_new_node() {
 1)   1.563 us    |                                      kstrdup_const();
 1)               |                                      kmem_cache_alloc() {
 1)   1.854 us    |                                        should_failslab();
 1)   1.646 us    |                                        memcg_kmem_put_cache();
 1)   8.937 us    |                                      }
 1)   1.562 us    |                                      _raw_spin_lock();
 1)   2.709 us    |                                      _raw_spin_unlock();
 1) + 23.896 us   |                                    }
 1)   1.563 us    |                                    kernfs_get();
 1) + 30.229 us   |                                  }
 1)               |                                  kernfs_add_one() {
 1)   1.562 us    |                                    mutex_lock();
 1)   1.708 us    |                                    kernfs_name_hash();
 1)   2.083 us    |                                    kernfs_link_sibling();
 1)   1.521 us    |                                    mutex_unlock();
 1)               |                                    kernfs_activate() {
 1)   1.666 us    |                                      mutex_lock();
 1)   2.417 us    |                                      kernfs_next_descendant_post();
 1)   1.792 us    |                                      kernfs_next_descendant_post();
 1)   1.562 us    |                                      mutex_unlock();
 1) + 15.792 us   |                                    }
 1) + 33.521 us   |                                  }
 1) + 68.645 us   |                                }
 1) + 72.020 us   |                              }
 1) + 75.500 us   |                            }
 1)   1.562 us    |                            kernfs_get();
 1) ! 680.625 us  |                          }
 1)   1.583 us    |                          mutex_unlock();
 1) ! 786.521 us  |                        }
 1) ! 790.354 us  |                      }
 1)               |                      irq_domain_associate() {
 1)               |                        irq_get_irq_data() {
 1)   1.666 us    |                          irq_to_desc();
 1)   4.834 us    |                        }
 1)   1.979 us    |                        mutex_lock();
 1)               |                        gpiochip_irq_map() {
 1)   1.771 us    |                          gpiochip_irqchip_irq_valid();
 1)               |                          irq_set_chip_data() {
 1)               |                            __irq_get_desc_lock() {
 1)   1.688 us    |                              _raw_spin_lock_irqsave();
 1)   4.937 us    |                            }
 1)               |                            __irq_put_desc_unlock() {
 1)   1.979 us    |                              _raw_spin_unlock_irqrestore();
 1)   5.625 us    |                            }
 1) + 15.625 us   |                          }
 1)   1.563 us    |                          irq_to_desc();
 1)               |                          irq_set_chip_and_handler_name() {
 1)               |                            irq_set_chip() {
 1)               |                              __irq_get_desc_lock() {
 1)   2.292 us    |                                _raw_spin_lock_irqsave();
 1)   6.771 us    |                              }
 1)               |                              __irq_put_desc_unlock() {
 1)   1.646 us    |                                _raw_spin_unlock_irqrestore();
 1)   4.854 us    |                              }
 1) + 16.666 us   |                            }
 1)               |                            __irq_set_handler() {
 1)               |                              __irq_get_desc_lock() {
 1)   2.021 us    |                                _raw_spin_lock_irqsave();
 1)   6.250 us    |                              }
 1)               |                              __irq_do_set_handler() {
 1)               |                                XXX_gpio_irq_ack [gpio_XXX]() {
 1)   1.604 us    |                                  gpiochip_get_data();
 1)   2.542 us    |                                  _raw_spin_lock();
 1)   1.917 us    |                                  _raw_spin_unlock();
 1) + 13.396 us   |                                }
 1) + 17.208 us   |                              }
 1)               |                              __irq_put_desc_unlock() {
 1)   1.604 us    |                                _raw_spin_unlock_irqrestore();
 1)   5.021 us    |                              }
 1) + 35.146 us   |                            }
 1) + 57.520 us   |                          }
 1)               |                          irq_modify_status() {
 1)               |                            __irq_get_desc_lock() {
 1)   1.688 us    |                              _raw_spin_lock_irqsave();
 1)   4.875 us    |                            }
 1)               |                            __irq_put_desc_unlock() {
 1)   1.979 us    |                              _raw_spin_unlock_irqrestore();
 1)   5.562 us    |                            } /* __irq_put_desc_unlock */
 1) + 15.770 us   |                          }
 1)               |                          irq_set_parent() {
 1)               |                            __irq_get_desc_lock() {
 1)   1.688 us    |                              _raw_spin_lock_irqsave();
 1)   4.958 us    |                            }
 1)               |                            __irq_put_desc_unlock() {
 1)   1.875 us    |                              _raw_spin_unlock_irqrestore();
 1)   5.417 us    |                            }
 1) + 15.771 us   |                          }
 1) ! 122.667 us  |                        }
 1)   1.604 us    |                        mutex_unlock();
 1)               |                        irq_modify_status() {
 1)               |                          __irq_get_desc_lock() {
 1)   1.730 us    |                            _raw_spin_lock_irqsave();
 1)   4.854 us    |                          }
 1)               |                          __irq_put_desc_unlock() {
 1)   1.667 us    |                            _raw_spin_unlock_irqrestore();
 1)   5.750 us    |                          }
 1) + 16.146 us   |                        }
 1) ! 158.479 us  |                      }
 1) # 1089.646 us |                    }
 1) # 1097.354 us |                  }
 1) # 1101.000 us |                }
 1) # 1104.375 us |              }
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.605 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.687 us    |                        should_failslab();
 1)   1.875 us    |                        memcg_kmem_put_cache();
 1)   9.542 us    |                      }
 1)   1.583 us    |                      _raw_spin_lock();
 1)   1.625 us    |                      _raw_spin_unlock();
 1) + 23.854 us   |                    }
 1)   1.875 us    |                    kernfs_get();
 1) + 30.875 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.562 us    |                    mutex_lock();
 1)   1.813 us    |                    kernfs_name_hash();
 1)   2.625 us    |                    kernfs_link_sibling();
 1)   1.708 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.584 us    |                      mutex_lock();
 1)   1.584 us    |                      kernfs_next_descendant_post();
 1)   1.521 us    |                      kernfs_next_descendant_post();
 1)   1.625 us    |                      mutex_unlock();
 1) + 14.583 us   |                    }
 1) + 32.230 us   |                  }
 1) + 68.167 us   |                }
 1) + 73.146 us   |              }
 1)   1.625 us    |              gpio_is_visible();
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.584 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.563 us    |                        should_failslab();
 1)   1.625 us    |                        memcg_kmem_put_cache();
 1)   8.375 us    |                      }
 1)   1.604 us    |                      _raw_spin_lock();
 1)   1.688 us    |                      _raw_spin_unlock();
 1) + 22.208 us   |                    }
 1)   1.771 us    |                    kernfs_get();
 1) + 28.895 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.605 us    |                    mutex_lock();
 1)   1.729 us    |                    kernfs_name_hash();
 1)   1.708 us    |                    kernfs_link_sibling();
 1)   1.604 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.646 us    |                      mutex_lock();
 1)   1.583 us    |                      kernfs_next_descendant_post();
 1)   1.562 us    |                      kernfs_next_descendant_post();
 1)   1.604 us    |                      mutex_unlock();
 1) + 14.542 us   |                    }
 1) + 31.021 us   |                  }
 1) + 64.875 us   |                }
 1) + 68.188 us   |              }
 1)   1.563 us    |              gpio_is_visible();
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.563 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.562 us    |                        should_failslab();
 1)               |                        __slab_alloc.isra.24() {
 1)   2.354 us    |                          ___slab_alloc();
 1)   5.916 us    |                        }
 1)   1.625 us    |                        memcg_kmem_put_cache();
 1) + 16.021 us   |                      }
 1)   1.605 us    |                      _raw_spin_lock();
 1)   1.605 us    |                      _raw_spin_unlock();
 1) + 29.645 us   |                    }
 1)   1.583 us    |                    kernfs_get();
 1) + 36.146 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.604 us    |                    mutex_lock();
 1)   1.750 us    |                    kernfs_name_hash();
 1)   1.979 us    |                    kernfs_link_sibling();
 1)   1.625 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.667 us    |                      mutex_lock();
 1)   1.584 us    |                      kernfs_next_descendant_post();
 1)   1.563 us    |                      kernfs_next_descendant_post();
 1)   1.583 us    |                      mutex_unlock();
 1) + 14.500 us   |                    }
 1) + 31.167 us   |                  }
 1) + 72.291 us   |                }
 1) + 75.583 us   |              }
 1)   1.583 us    |              kernfs_put();
 1) # 1429.958 us |            }
 1) # 1433.333 us |          }
 1)   1.605 us    |          bus_add_device();
 1)               |          dpm_sysfs_add() {
 1)               |            sysfs_create_group() {
 1)               |              internal_create_group() {
 1)   1.625 us    |                device_get_ownership();
 1)               |                kernfs_create_dir_ns() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.583 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.605 us    |                        should_failslab();
 1)   1.625 us    |                        memcg_kmem_put_cache();
 1) + 10.125 us   |                      }
 1)   1.625 us    |                      _raw_spin_lock();
 1)   1.562 us    |                      _raw_spin_unlock();
 1) + 23.917 us   |                    }
 1)   1.604 us    |                    kernfs_get();
 1) + 30.396 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.604 us    |                    mutex_lock();
 1)   1.708 us    |                    kernfs_name_hash();
 1)   2.021 us    |                    kernfs_link_sibling();
 1)   1.604 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.604 us    |                      mutex_lock();
 1)   1.604 us    |                      kernfs_next_descendant_post();
 1)   1.542 us    |                      kernfs_next_descendant_post();
 1)   1.562 us    |                      mutex_unlock();
 1) + 14.355 us   |                    }
 1) + 30.979 us   |                  }
 1) + 66.250 us   |                }
 1)   1.584 us    |                kernfs_get();
 1)   1.750 us    |                kernfs_put();
 1) + 79.584 us   |              }
 1) + 82.792 us   |            }
 1)               |            sysfs_merge_group() {
 1)               |              kernfs_find_and_get_ns() {
 1)   1.583 us    |                mutex_lock();
 1)               |                kernfs_find_ns() {
 1)   1.604 us    |                  kernfs_name_hash();
 1)   5.063 us    |                }
 1)   1.562 us    |                kernfs_get();
 1)   1.563 us    |                mutex_unlock();
 1) + 17.875 us   |              }
 1)   1.563 us    |              device_get_ownership();
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.563 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.542 us    |                        should_failslab();
 1)   1.583 us    |                        memcg_kmem_put_cache();
 1)   8.270 us    |                      }
 1)   1.562 us    |                      _raw_spin_lock();
 1)   1.666 us    |                      _raw_spin_unlock();
 1) + 21.625 us   |                    }
 1)   1.562 us    |                    kernfs_get();
 1) + 28.145 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.583 us    |                    mutex_lock();
 1)   1.771 us    |                    kernfs_name_hash();
 1)   1.563 us    |                    kernfs_link_sibling();
 1)   1.563 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.583 us    |                      mutex_lock();
 1)   1.563 us    |                      kernfs_next_descendant_post();
 1)   1.625 us    |                      kernfs_next_descendant_post();
 1)   1.563 us    |                      mutex_unlock();
 1) + 14.479 us   |                    }
 1) + 30.750 us   |                  }
 1) + 63.895 us   |                }
 1) + 67.146 us   |              }
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.563 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.563 us    |                        should_failslab();
 1)   1.562 us    |                        memcg_kmem_put_cache();
 1)   8.166 us    |                      }
 1)   1.583 us    |                      _raw_spin_lock();
 1)   1.604 us    |                      _raw_spin_unlock();
 1) + 21.562 us   |                    }
 1)   1.521 us    |                    kernfs_get();
 1) + 29.354 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.542 us    |                    mutex_lock();
 1)   1.730 us    |                    kernfs_name_hash();
 1)   1.709 us    |                    kernfs_link_sibling();
 1)   1.542 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.792 us    |                      mutex_lock();
 1)   1.562 us    |                      kernfs_next_descendant_post();
 1)   1.563 us    |                      kernfs_next_descendant_post();
 1)   1.604 us    |                      mutex_unlock();
 1) + 14.645 us   |                    }
 1) + 31.104 us   |                  }
 1) + 65.312 us   |                }
 1) + 68.562 us   |              }
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.625 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.521 us    |                        should_failslab();
 1)   1.562 us    |                        memcg_kmem_put_cache();
 1)   8.270 us    |                      }
 1)   1.583 us    |                      _raw_spin_lock();
 1)   1.667 us    |                      _raw_spin_unlock();
 1) + 21.750 us   |                    }
 1)   1.563 us    |                    kernfs_get();
 1) + 28.083 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.562 us    |                    mutex_lock();
 1)   1.812 us    |                    kernfs_name_hash();
 1)   1.729 us    |                    kernfs_link_sibling();
 1)   1.562 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.583 us    |                      mutex_lock();
 1)   1.563 us    |                      kernfs_next_descendant_post();
 1)   1.563 us    |                      kernfs_next_descendant_post();
 1)   1.646 us    |                      mutex_unlock();
 1) + 14.521 us   |                    }
 1) + 30.979 us   |                  }
 1) + 63.813 us   |                }
 1) + 66.979 us   |              }
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.563 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.563 us    |                        should_failslab();
 1)   1.562 us    |                        memcg_kmem_put_cache();
 1)   8.125 us    |                      }
 1)   1.583 us    |                      _raw_spin_lock();
 1)   1.646 us    |                      _raw_spin_unlock();
 1) + 21.541 us   |                    }
 1)   1.563 us    |                    kernfs_get();
 1) + 27.875 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.521 us    |                    mutex_lock();
 1)   1.833 us    |                    kernfs_name_hash();
 1)   1.687 us    |                    kernfs_link_sibling();
 1)   1.562 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.584 us    |                      mutex_lock();
 1)   1.562 us    |                      kernfs_next_descendant_post();
 1)   1.563 us    |                      kernfs_next_descendant_post();
 1)   1.583 us    |                      mutex_unlock();
 1) + 14.479 us   |                    }
 1) + 30.980 us   |                  }
 1) + 63.646 us   |                }
 1) + 66.834 us   |              }
 1)               |              sysfs_add_file_mode_ns() {
 1)               |                __kernfs_create_file() {
 1)               |                  kernfs_new_node() {
 1)               |                    __kernfs_new_node() {
 1)   1.563 us    |                      kstrdup_const();
 1)               |                      kmem_cache_alloc() {
 1)   1.563 us    |                        should_failslab();
 1)   1.563 us    |                        memcg_kmem_put_cache();
 1)   8.292 us    |                      }
 1)   1.583 us    |                      _raw_spin_lock();
 1)   1.625 us    |                      _raw_spin_unlock();
 1) + 23.312 us   |                    }
 1)   1.562 us    |                    kernfs_get();
 1) + 29.730 us   |                  }
 1)               |                  kernfs_add_one() {
 1)   1.563 us    |                    mutex_lock();
 1)   1.812 us    |                    kernfs_name_hash();
 1)   1.646 us    |                    kernfs_link_sibling();
 1)   1.605 us    |                    mutex_unlock();
 1)               |                    kernfs_activate() {
 1)   1.604 us    |                      mutex_lock();
 1)   1.562 us    |                      kernfs_next_descendant_post();
 1)   1.625 us    |                      kernfs_next_descendant_post();
 1)   1.583 us    |                      mutex_unlock();
 1) + 14.479 us   |                    }
 1) + 31.041 us   |                  }
 1) + 65.521 us   |                }
 1) + 68.708 us   |              }
 1)   1.604 us    |              kernfs_put();
 1) ! 374.584 us  |            }
 1) ! 462.396 us  |          }
 1)               |          device_pm_add() {
 1)               |            device_pm_check_callbacks() {
 1)   1.688 us    |              _raw_spin_lock_irq();
 1)   1.604 us    |              pm_ops_is_empty();
 1)   1.666 us    |              _raw_spin_unlock_irq();
 1) + 11.708 us   |            }
 1)   1.729 us    |            mutex_lock();
 1)   1.646 us    |            mutex_unlock();
 1) + 22.271 us   |          }
 1)   1.625 us    |          dev_uevent_filter();
 1)   1.604 us    |          dev_uevent_name();
 1)               |          kmem_cache_alloc_trace() {
 1)   1.562 us    |            should_failslab();
 1)   1.625 us    |            memcg_kmem_put_cache();
 1)   10.000 us   |          }
 1)               |          __kmalloc() {
 1)   1.562 us    |            kmalloc_slab();
 1)   1.563 us    |            should_failslab();
 1)   1.604 us    |            memcg_kmem_put_cache();
 1) + 11.708 us   |          }
 1)               |          dev_uevent() {
 1)   1.667 us    |            of_device_uevent();
 1)   5.583 us    |          }
 1)   1.646 us    |          mutex_lock();
 1)   1.625 us    |          class_dir_child_ns_type();
 1)               |          netlink_has_listeners() {
 1)   1.562 us    |            __rcu_read_lock();
 1)   1.563 us    |            __rcu_read_unlock();
 1)   8.459 us    |          }
 1)               |          __alloc_skb() {
 1)               |            kmem_cache_alloc_node() {
 1)   1.583 us    |              should_failslab();
 1)   1.583 us    |              memcg_kmem_put_cache();
 1)   8.541 us    |            }
 1)               |            __kmalloc_reserve.isra.8() {
 1)               |              __kmalloc_node_track_caller() {
 1)   1.708 us    |                kmalloc_slab();
 1)   1.562 us    |                should_failslab();
 1)   1.604 us    |                memcg_kmem_put_cache();
 1) + 11.646 us   |              }
 1) + 14.875 us   |            }
 1)   1.646 us    |            ksize();
 1) + 31.833 us   |          }
 1)   1.583 us    |          skb_put();
 1)   1.646 us    |          skb_put();
 1)               |          netlink_broadcast() {
 1)               |            netlink_broadcast_filtered() {
 1)   1.771 us    |              netlink_trim();
 1)   1.625 us    |              _raw_read_lock();
 1)   1.771 us    |              _raw_read_unlock();
 1)               |              skb_clone() {
 1)               |                kmem_cache_alloc() {
 1)   1.604 us    |                  should_failslab();
 1)   1.645 us    |                  memcg_kmem_put_cache();
 1)   8.313 us    |                }
 1)               |                __skb_clone() {
 1)   1.687 us    |                  __copy_skb_header();
 1)   4.980 us    |                }
 1) + 18.250 us   |              }
 1)               |              sk_filter_trim_cap() {
 1)   1.562 us    |                __rcu_read_lock();
 1)   1.583 us    |                __rcu_read_unlock();
 1)   8.229 us    |              }
 1)               |              peernet2id() {
 1)   1.604 us    |                _raw_spin_lock_bh();
 1)   2.375 us    |                __peernet2id_alloc();
 1)               |                _raw_spin_unlock_bh() {
 1)   1.729 us    |                  __local_bh_enable_ip();
 1)   5.000 us    |                }
 1) + 15.812 us   |              }
 1)   1.708 us    |              netlink_skb_set_owner_r();
 1)               |              __netlink_sendskb() {
 1)               |                netlink_deliver_tap() {
 1)   1.645 us    |                  __rcu_read_lock();
 1)   1.625 us    |                  __rcu_read_unlock();
 1)   1.583 us    |                  __rcu_read_lock();
 1)   1.583 us    |                  __rcu_read_unlock();
 1) + 14.688 us   |                }
 1)               |                skb_queue_tail() {
 1)   1.750 us    |                  _raw_spin_lock_irqsave();
 1)   1.771 us    |                  _raw_spin_unlock_irqrestore();
 1)   8.375 us    |                }
 1)               |                sock_def_readable() {
 1)   1.584 us    |                  __rcu_read_lock();
 1)               |                  __wake_up_sync_key() {
 1)               |                    __wake_up_common_lock() {
 1)   1.709 us    |                      _raw_spin_lock_irqsave();
 1)               |                      __wake_up_common() {
 1)               |                        ep_poll_callback() {
 1)   1.750 us    |                          _raw_spin_lock_irqsave();
 1)   1.833 us    |                          __rcu_read_lock();
 1)   1.604 us    |                          __rcu_read_unlock();
 1)               |                          __wake_up_locked() {
 1)               |                            __wake_up_common() {
 1)               |                              default_wake_function() {
 1)               |                                try_to_wake_up() {
 1)   1.813 us    |                                  _raw_spin_lock_irqsave();
 1)               |                                  select_task_rq_fair() {
 1)   1.605 us    |                                    __rcu_read_lock();
 1)               |                                    select_idle_sibling() {
 1)   1.625 us    |                                      available_idle_cpu();
 1)   1.625 us    |                                      available_idle_cpu();
 1)   1.562 us    |                                      available_idle_cpu();
 1) + 12.480 us   |                                    }
 1)   1.583 us    |                                    __rcu_read_unlock();
 1) + 22.583 us   |                                  }
 1)               |                                  smp_send_reschedule() {
 1)               |                                    smp_cross_call() {
 1)   2.125 us    |                                      gic_raise_softirq();
 1)   5.541 us    |                                    }
 1)   8.688 us    |                                  }
 1)   1.688 us    |                                  _raw_spin_unlock_irqrestore();
 1) + 44.208 us   |                                }
 1) + 47.416 us   |                              }
 1) + 51.291 us   |                            }
 1) + 54.541 us   |                          }
 1)   1.687 us    |                          _raw_spin_unlock_irqrestore();
 1) + 71.916 us   |                        }
 1) + 75.500 us   |                      }
 1)   1.666 us    |                      _raw_spin_unlock_irqrestore();
 1) + 85.562 us   |                    }
 1) + 88.750 us   |                  }
 1)   1.667 us    |                  __rcu_read_unlock();
 1) + 98.646 us   |                }
 1) ! 128.542 us  |              }
 1)   3.125 us    |              consume_skb();
 1)               |              __wake_up() {
 1)               |                __wake_up_common_lock() {
 1)   1.708 us    |                  _raw_spin_lock_irqsave();
 1)   1.646 us    |                  __wake_up_common();
 1)   1.708 us    |                  _raw_spin_unlock_irqrestore();
 1) + 11.458 us   |                }
 1) + 14.688 us   |              }
 1)   1.667 us    |              consume_skb();
 1) ! 225.875 us  |            }
 1) ! 230.604 us  |          }
 1)               |          netlink_has_listeners() {
 1)   1.583 us    |            __rcu_read_lock();
 1)   1.604 us    |            __rcu_read_unlock();
 1)   8.021 us    |          }
 1)               |          netlink_broadcast() {
 1)               |            netlink_broadcast_filtered() {
 1)   1.563 us    |              netlink_trim();
 1)   1.563 us    |              _raw_read_lock();
 1)   1.625 us    |              _raw_read_unlock();
 1)   1.771 us    |              consume_skb();
 1)               |              __wake_up() {
 1)               |                __wake_up_common_lock() {
 1)   1.750 us    |                  _raw_spin_lock_irqsave();
 1)   1.563 us    |                  __wake_up_common();
 1)   1.666 us    |                  _raw_spin_unlock_irqrestore();
 1) + 11.459 us   |                }
 1) + 14.583 us   |              }
 1)   1.541 us    |              consume_skb();
 1) + 36.708 us   |            }
 1) + 40.000 us   |          }
 1)               |          consume_skb() {
 1)               |            skb_release_all() {
 1)   1.708 us    |              skb_release_head_state();
 1)   1.667 us    |              skb_release_data();
 1)   8.187 us    |            }
 1)               |            kfree_skbmem() {
 1)   1.792 us    |              kmem_cache_free();
 1)   4.958 us    |            }
 1) + 17.958 us   |          }
 1)   1.563 us    |          mutex_unlock();
 1)   1.583 us    |          class_dir_child_ns_type();
 1)               |          call_usermodehelper_setup() {
 1)               |            kmem_cache_alloc_trace() {
 1)   1.625 us    |              should_failslab();
 1)   1.604 us    |              memcg_kmem_put_cache();
 1)   8.333 us    |            }
 1) + 11.583 us   |          }
 1)               |          call_usermodehelper_exec() {
 1)               |            queue_work_on() {
 1)               |              __queue_work() {
 1)   1.563 us    |                get_work_pool();
 1)   1.625 us    |                _raw_spin_lock();
 1)               |                insert_work() {
 1)               |                  wake_up_process() {
 1)               |                    try_to_wake_up() {
 1)   1.813 us    |                      _raw_spin_lock_irqsave();
 1)               |                      select_task_rq_fair() {
 1)   1.666 us    |                        __rcu_read_lock();
 1)               |                        select_idle_sibling() {
 1)   1.771 us    |                          available_idle_cpu();
 1)   5.000 us    |                        }
 1)   1.541 us    |                        __rcu_read_unlock();
 1) + 14.938 us   |                      }
 1)               |                      smp_send_reschedule() {
 1)               |                        smp_cross_call() {
 1)   2.063 us    |                          gic_raise_softirq();
 1)   5.230 us    |                        }
 1)   8.375 us    |                      }
 1)   1.604 us    |                      _raw_spin_unlock_irqrestore();
 1) + 35.792 us   |                    }
 1) + 38.937 us   |                  }
 1) + 42.459 us   |                }
 1)   1.708 us    |                _raw_spin_unlock();
 1) + 56.041 us   |              }
 1) + 59.583 us   |            }
 1)               |            __wake_up() {
 1)               |              __wake_up_common_lock() {
 1)   1.771 us    |                _raw_spin_lock_irqsave();
 1)   1.563 us    |                __wake_up_common();
 1)   1.687 us    |                _raw_spin_unlock_irqrestore();
 1) + 12.917 us   |              }
 1) + 16.250 us   |            }
 1) + 81.229 us   |          }
 1)   2.146 us    |          kfree();
 1)   1.625 us    |          kfree();
 1)   1.688 us    |          bus_probe_device();
 1)   1.583 us    |          klist_children_get();
 1)   1.584 us    |          _raw_spin_lock();
 1)   1.583 us    |          _raw_spin_unlock();
 1)   1.604 us    |          mutex_lock();
 1)               |          klist_class_dev_get() {
 1)   1.750 us    |            get_device();
 1)   4.937 us    |          }
 1)   1.584 us    |          _raw_spin_lock();
 1)   1.563 us    |          _raw_spin_unlock();
 1)   1.625 us    |          mutex_unlock();
 1) # 3820.959 us |        }
 1) # 3904.271 us |      }
 1) # 3907.542 us |    }
 1)   1.646 us    |    mutex_unlock();
 1) # 3947.250 us |  }

// 实验2
/sys/class/gpio/gpio480# cat direction 
in

// 查看结果
 2)               |  gpiod_get_direction() {
 2)   ==========> |
 2)               |    gic_handle_irq() {
 2)               |      handle_IPI() {
 2)               |        irq_enter() {
 2)               |          rcu_irq_enter() {
 2)               |            rcu_nmi_enter() {
 2)   1.604 us    |              rcu_dynticks_curr_cpu_in_eqs();
 2)   5.416 us    |            }
 2)   8.854 us    |          }
 2)   1.854 us    |          irqtime_account_irq();
 2) + 15.937 us   |        }
 2)               |        __wake_up() {
 2)               |          __wake_up_common_lock() {
 2)   1.688 us    |            _raw_spin_lock_irqsave();
 2)               |            __wake_up_common() {
 2)               |              autoremove_wake_function() {
 2)               |                default_wake_function() {
 2)               |                  try_to_wake_up() {
 2)   1.708 us    |                    _raw_spin_lock_irqsave();
 2)               |                    select_task_rq_fair() {
 2)   1.625 us    |                      __rcu_read_lock();
 2)               |                      select_idle_sibling() {
 2)   1.563 us    |                        available_idle_cpu();
 2)   1.521 us    |                        available_idle_cpu();
 2)   1.604 us    |                        available_idle_cpu();
 2) + 11.750 us   |                      }
 2)   1.562 us    |                      __rcu_read_unlock();
 2) + 21.979 us   |                    }
 2)               |                    set_task_cpu() {
 2)               |                      migrate_task_rq_fair() {
 2)               |                        remove_entity_load_avg() {
 2)   1.687 us    |                          __update_load_avg_blocked_se();
 2)   1.687 us    |                          _raw_spin_lock_irqsave();
 2)   1.667 us    |                          _raw_spin_unlock_irqrestore();
 2) + 11.583 us   |                        }
 2) + 15.104 us   |                      }
 2)   1.542 us    |                      set_task_rq_fair();
 2) + 22.146 us   |                    }
 2)   1.625 us    |                    _raw_spin_lock();
 2)   1.750 us    |                    update_rq_clock.part.17();
 2)               |                    ttwu_do_activate.isra.20() {
 2)               |                      activate_task() {
 2)               |                        enqueue_task_fair() {
 2)   1.583 us    |                          update_curr();
 2)               |                          __update_load_avg_cfs_rq() {
 2)   1.584 us    |                            __accumulate_pelt_segments();
 2)   4.958 us    |                          }
 2)               |                          attach_entity_load_avg() {
 2)               |                            dbs_update_util_handler() {
 2)   1.562 us    |                              cpufreq_this_cpu_can_update();
 2)               |                              arch_irq_work_raise() {
 2)               |                                smp_cross_call() {
 2)   1.875 us    |                                  gic_raise_softirq();
 2)   5.000 us    |                                }
 2)   8.313 us    |                              }
 2) + 15.584 us   |                            }
 2) + 19.000 us   |                          }
 2)   1.562 us    |                          update_cfs_group();
 2)   1.854 us    |                          account_entity_enqueue();
 2)   1.563 us    |                          __enqueue_entity();
 2)   1.583 us    |                          update_curr();
 2)               |                          __update_load_avg_se() {
 2)   1.541 us    |                            __accumulate_pelt_segments();
 2)   5.104 us    |                          }
 2)               |                          __update_load_avg_cfs_rq() {
 2)   1.625 us    |                            __accumulate_pelt_segments();
 2)   4.938 us    |                          }
 2)               |                          dbs_update_util_handler() {
 2)   1.562 us    |                            cpufreq_this_cpu_can_update();
 2)   4.688 us    |                          }
 2)               |                          update_cfs_group() {
 2)   1.646 us    |                            reweight_entity();
 2)   4.854 us    |                          }
 2)   1.625 us    |                          account_entity_enqueue();
 2)   1.563 us    |                          __enqueue_entity();
 2) + 79.563 us   |                        }
 2) + 83.062 us   |                      }
 2)               |                      ttwu_do_wakeup.isra.19() {
 2)               |                        check_preempt_curr() {
 2)               |                          resched_curr() {
 2)   1.584 us    |                            set_nr_and_not_polling();
 2)               |                            smp_send_reschedule() {
 2)               |                              smp_cross_call() {
 2)   1.771 us    |                                gic_raise_softirq();
 2)   4.937 us    |                              }
 2)   8.063 us    |                            }
 2) + 14.437 us   |                          }
 2) + 20.083 us   |                        }
 2) + 25.583 us   |                      }
 2) ! 113.709 us  |                    }
 2)   1.708 us    |                    _raw_spin_unlock();
 2)   1.667 us    |                    _raw_spin_unlock_irqrestore();
 2) ! 181.792 us  |                  }
 2) ! 184.938 us  |                }
 2) ! 188.229 us  |              }
 2) ! 191.958 us  |            }
 2)   1.625 us    |            _raw_spin_unlock_irqrestore();
 2) ! 201.813 us  |          }
 2) ! 205.146 us  |        }
 2)               |        irq_exit() {
 2)               |          irqtime_account_irq() {
 2)   1.562 us    |            irqtime_account_delta.isra.0();
 2)   4.980 us    |          }
 2)   1.750 us    |          idle_cpu();
 2)               |          rcu_irq_exit() {
 2)               |            rcu_nmi_exit() {
 2)   1.562 us    |              rcu_dynticks_curr_cpu_in_eqs();
 2)   4.833 us    |            }
 2)   8.250 us    |          }
 2) + 21.687 us   |        }
 2) ! 250.833 us  |      }
 2)               |      handle_IPI() {
 2)               |        irq_enter() {
 2)               |          rcu_irq_enter() {
 2)               |            rcu_nmi_enter() {
 2)   1.562 us    |              rcu_dynticks_curr_cpu_in_eqs();
 2)   4.750 us    |            }
 2)   8.334 us    |          }
 2)   1.771 us    |          irqtime_account_irq();
 2) + 14.791 us   |        }
 2)               |        __wake_up() {
 2)               |          __wake_up_common_lock() {
 2)   1.687 us    |            _raw_spin_lock_irqsave();
 2)   1.625 us    |            __wake_up_common();
 2)   1.604 us    |            _raw_spin_unlock_irqrestore();
 2) + 11.708 us   |          }
 2) + 15.166 us   |        }
 2)               |        dbs_irq_work() {
 2)               |          queue_work_on() {
 2)               |            __queue_work() {
 2)   1.875 us    |              get_work_pool();
 2)   1.605 us    |              _raw_spin_lock();
 2)   1.583 us    |              _raw_spin_unlock();
 2)   1.584 us    |              _raw_spin_lock();
 2)               |              insert_work() {
 2)               |                wake_up_process() {
 2)               |                  try_to_wake_up() {
 2)   1.687 us    |                    _raw_spin_lock_irqsave();
 2)   1.729 us    |                    _raw_spin_lock();
 2)               |                    update_rq_clock.part.17() {
 2)   2.020 us    |                      update_irq_load_avg();
 2)   5.438 us    |                    }
 2)               |                    ttwu_do_activate.isra.20() {
 2)               |                      activate_task() {
 2)               |                        enqueue_task_fair() {
 2)               |                          update_curr() {
 2)   1.583 us    |                            __calc_delta();
 2)   1.688 us    |                            update_min_vruntime();
 2)   8.375 us    |                          } /* update_curr */
 2)               |                          __update_load_avg_se() {
 2)   1.542 us    |                            __accumulate_pelt_segments();
 2)   4.813 us    |                          }
 2)               |                          __update_load_avg_cfs_rq() {
 2)   1.667 us    |                            __accumulate_pelt_segments();
 2)   4.980 us    |                          }
 2)               |                          dbs_update_util_handler() {
 2)   1.708 us    |                            cpufreq_this_cpu_can_update();
 2)   5.105 us    |                          }
 2)   1.562 us    |                          update_cfs_group();
 2)   1.625 us    |                          account_entity_enqueue();
 2)   1.562 us    |                          __enqueue_entity();
 2) + 42.021 us   |                        }
 2) + 45.396 us   |                      }
 2)               |                      wq_worker_waking_up() {
 2)   1.563 us    |                        kthread_data();
 2)   4.833 us    |                      }
 2)               |                      ttwu_do_wakeup.isra.19() {
 2)               |                        check_preempt_curr() {
 2)               |                          check_preempt_wakeup() {
 2)   1.646 us    |                            update_curr();
 2)   1.562 us    |                            wakeup_preempt_entity.isra.9();
 2)   1.833 us    |                            resched_curr();
 2) + 13.334 us   |                          }
 2) + 16.729 us   |                        }
 2) + 20.375 us   |                      }
 2) + 77.333 us   |                    }
 2)   1.625 us    |                    _raw_spin_unlock();
 2)   1.667 us    |                    _raw_spin_unlock_irqrestore();
 2) ! 101.188 us  |                  }
 2) ! 104.312 us  |                }
 2) ! 108.083 us  |              }
 2)   1.604 us    |              _raw_spin_unlock();
 2) ! 128.854 us  |            }
 2) ! 132.291 us  |          }
 2) ! 135.438 us  |        }
 2)               |        irq_exit() {
 2)               |          irqtime_account_irq() {
 2)   1.604 us    |            irqtime_account_delta.isra.0();
 2)   5.042 us    |          }
 2)   2.188 us    |          idle_cpu();
 2)               |          rcu_irq_exit() {
 2)               |            rcu_nmi_exit() {
 2)   1.875 us    |              rcu_dynticks_curr_cpu_in_eqs();
 2)   5.979 us    |            }
 2)   9.708 us    |          }
 2) + 23.896 us   |        }
 2) ! 198.292 us  |      }
 2)               |      __handle_domain_irq() {
 2)               |        irq_enter() {
 2)               |          rcu_irq_enter() {
 2)               |            rcu_nmi_enter() {
 2)   1.792 us    |              rcu_dynticks_curr_cpu_in_eqs();
 2)   5.271 us    |            }
 2)   8.854 us    |          }
 2)   2.104 us    |          irqtime_account_irq();
 2) + 16.459 us   |        }
 2)               |        irq_find_mapping() {
 2)   1.625 us    |          __rcu_read_lock();
 2)   1.563 us    |          __rcu_read_unlock();
 2)   8.688 us    |        }
 2)               |        generic_handle_irq() {
 2)               |          handle_percpu_devid_irq() {
 2)               |            arch_timer_handler_phys() {
 2)               |              hrtimer_interrupt() {
 2)   1.687 us    |                _raw_spin_lock_irqsave();
 2)               |                ktime_get_update_offsets_now() {
 2)   1.729 us    |                  arch_counter_read();
 2)   5.208 us    |                }
 2)               |                __hrtimer_run_queues() {
 2)   2.084 us    |                  __remove_hrtimer();
 2)   1.645 us    |                  _raw_spin_unlock_irqrestore();
 2)               |                  tick_sched_timer() {
 2)               |                    ktime_get() {
 2)   1.667 us    |                      arch_counter_read();
 2)   4.958 us    |                    }
 2)   1.646 us    |                    tick_sched_do_timer();
 2)               |                    tick_sched_handle.isra.5() {
 2)               |                      update_process_times() {
 2)               |                        account_process_tick() {
 2)               |                          irqtime_account_process_tick.isra.1() {
 2)               |                            account_system_index_time() {
 2)               |                              cpuacct_account_field() {
 2)   1.562 us    |                                __rcu_read_lock();
 2)   1.542 us    |                                __rcu_read_unlock();
 2)   8.063 us    |                              }
 2)   1.646 us    |                              __rcu_read_lock();
 2)               |                              __cgroup_account_cputime_field() {
 2)   1.583 us    |                                cgroup_base_stat_cputime_account_begin();
 2)               |                                cgroup_base_stat_cputime_account_end.isra.0() {
 2)   1.604 us    |                                  cgroup_rstat_updated();
 2)   4.791 us    |                                }
 2) + 11.146 us   |                              }
 2)   1.729 us    |                              __rcu_read_unlock();
 2)               |                              acct_account_cputime() {
 2)   1.688 us    |                                __acct_update_integrals();
 2)   4.958 us    |                              }
 2) + 37.771 us   |                            }
 2) + 41.104 us   |                          }
 2) + 44.333 us   |                        }
 2)               |                        run_local_timers() {
 2)   1.834 us    |                          hrtimer_run_queues();
 2)               |                          raise_softirq() {
 2)   1.541 us    |                            __raise_softirq_irqoff();
 2)   4.980 us    |                          }
 2) + 11.687 us   |                        }
 2)               |                        rcu_check_callbacks() {
 2)   1.646 us    |                          rcu_is_cpu_rrupt_from_idle();
 2)   1.959 us    |                          rcu_bh_qs();
 2)   1.604 us    |                          rcu_preempt_qs();
 2)   1.584 us    |                          rcu_segcblist_ready_cbs();
 2)               |                          invoke_rcu_core() {
 2)               |                            raise_softirq() {
 2)   1.542 us    |                              __raise_softirq_irqoff();
 2)   4.916 us    |                            }
 2)   8.084 us    |                          }
 2) + 27.125 us   |                        }
 2)               |                        scheduler_tick() {
 2)   1.646 us    |                          _raw_spin_lock();
 2)               |                          update_rq_clock.part.17() {
 2)   1.750 us    |                            update_irq_load_avg();
 2)   5.521 us    |                          }
 2)               |                          task_tick_fair() {
 2)               |                            update_curr() {
 2)   1.583 us    |                              update_min_vruntime();
 2)               |                              cpuacct_charge() {
 2)   1.562 us    |                                __rcu_read_lock();
 2)   1.562 us    |                                __rcu_read_unlock();
 2)   8.084 us    |                              }
 2)   1.542 us    |                              __rcu_read_lock();
 2)               |                              __cgroup_account_cputime() {
 2)   1.771 us    |                                cgroup_base_stat_cputime_account_begin();
 2)               |                                cgroup_base_stat_cputime_account_end.isra.0() {
 2)   1.625 us    |                                  cgroup_rstat_updated();
 2)   4.750 us    |                                }
 2) + 11.395 us   |                              }
 2)   1.855 us    |                              __rcu_read_unlock();
 2) + 35.041 us   |                            }
 2)               |                            __update_load_avg_se() {
 2)   1.583 us    |                              __accumulate_pelt_segments();
 2)   4.979 us    |                            }
 2)               |                            __update_load_avg_cfs_rq() {
 2)   1.583 us    |                              __accumulate_pelt_segments();
 2)   4.979 us    |                            }
 2)   1.646 us    |                            update_cfs_group();
 2)   1.563 us    |                            hrtimer_active();
 2)               |                            update_curr() {
 2)   1.562 us    |                              __calc_delta();
 2)   1.583 us    |                              update_min_vruntime();
 2)   8.333 us    |                            }
 2)               |                            __update_load_avg_se() {
 2)   1.562 us    |                              __accumulate_pelt_segments();
 2)   4.854 us    |                            }
 2)   1.625 us    |                            __update_load_avg_cfs_rq();
 2)               |                            update_cfs_group() {
 2)               |                              reweight_entity() {
 2)   1.646 us    |                                update_curr();
 2)   1.625 us    |                                account_entity_dequeue();
 2)   1.646 us    |                                account_entity_enqueue();
 2) + 12.041 us   |                              }
 2) + 15.458 us   |                            }
 2)   1.583 us    |                            hrtimer_active();
 2)               |                            sched_slice.isra.13() {
 2)   1.708 us    |                              __calc_delta();
 2)   5.312 us    |                            }
 2)   1.563 us    |                            resched_curr();
 2) ! 108.958 us  |                          }
 2)               |                          cpu_load_update_active() {
 2)   1.562 us    |                            tick_nohz_tick_stopped();
 2)   1.709 us    |                            cpu_load_update();
 2)   8.313 us    |                          }
 2)   1.542 us    |                          calc_global_load_tick();
 2)   1.583 us    |                          _raw_spin_unlock();
 2)               |                          trigger_load_balance() {
 2)   1.583 us    |                            nohz_balance_exit_idle();
 2)               |                            kick_ilb() {
 2)   1.541 us    |                              housekeeping_cpumask();
 2)   1.729 us    |                              idle_cpu();
 2)               |                              smp_send_reschedule() {
 2)               |                                smp_cross_call() {
 2)   1.980 us    |                                  gic_raise_softirq();
 2)   5.104 us    |                                }
 2)   8.333 us    |                              }
 2) + 18.750 us   |                            }
 2) + 25.729 us   |                          }
 2) ! 167.375 us  |                        }
 2)   1.812 us    |                        run_posix_cpu_timers();
 2) ! 263.895 us  |                      }
 2)   1.875 us    |                      profile_tick();
 2) ! 270.562 us  |                    }
 2)   1.604 us    |                    hrtimer_forward();
 2) ! 287.458 us  |                  }
 2)   1.645 us    |                  _raw_spin_lock_irq();
 2)   1.813 us    |                  enqueue_hrtimer();
 2) ! 304.771 us  |                }
 2)               |                __hrtimer_get_next_event() {
 2)   1.896 us    |                  __hrtimer_next_event_base();
 2)   1.750 us    |                  __hrtimer_next_event_base();
 2)   8.855 us    |                }
 2)   1.666 us    |                _raw_spin_unlock_irqrestore();
 2)               |                tick_program_event() {
 2)               |                  clockevents_program_event() {
 2)               |                    ktime_get() {
 2)   1.666 us    |                      arch_counter_read();
 2)   4.834 us    |                    } /* ktime_get */
 2)   1.709 us    |                    arch_timer_set_next_event_phys();
 2) + 11.625 us   |                  }
 2) + 15.729 us   |                }
 2) ! 349.792 us  |              }
 2) ! 353.083 us  |            }
 2)   1.750 us    |            gic_eoimode1_eoi_irq();
 2) ! 359.709 us  |          }
 2) ! 363.479 us  |        }
 2)               |        irq_exit() {
 2)               |          irqtime_account_irq() {
 2)   1.625 us    |            irqtime_account_delta.isra.0();
 2)   5.479 us    |          }
 2)   1.916 us    |          ksoftirqd_running();
 2)               |          __do_softirq() {
 2)   1.937 us    |            irqtime_account_irq();
 2)               |            run_timer_softirq() {
 2)   1.812 us    |              _raw_spin_lock_irq();
 2)   1.583 us    |              collect_expired_timers();
 2)   1.625 us    |              _raw_spin_unlock_irq();
 2)   1.646 us    |              _raw_spin_lock_irq();
 2)   1.584 us    |              collect_expired_timers();
 2)   1.667 us    |              _raw_spin_unlock_irq();
 2) + 22.542 us   |            }
 2)               |            rcu_process_callbacks() {
 2)               |              note_gp_changes() {
 2)   1.750 us    |                _raw_spin_trylock();
 2)               |                __note_gp_changes() {
 2)               |                  rcu_advance_cbs() {
 2)   1.562 us    |                    rcu_segcblist_pend_cbs();
 2)   1.605 us    |                    rcu_segcblist_advance();
 2)               |                    rcu_accelerate_cbs() {
 2)   1.771 us    |                      rcu_segcblist_pend_cbs();
 2)   1.563 us    |                      rcu_segcblist_accelerate();
 2)   8.375 us    |                    }
 2) + 18.146 us   |                  }
 2) + 21.521 us   |                }
 2)   1.604 us    |                _raw_spin_unlock_irqrestore();
 2) + 31.875 us   |              }
 2)   1.584 us    |              rcu_jiffies_till_stall_check();
 2)   1.604 us    |              rcu_segcblist_ready_cbs();
 2)   1.562 us    |              rcu_segcblist_ready_cbs();
 2)   1.750 us    |              rcu_segcblist_extract_done_cbs();
 2)   1.687 us    |              rcu_cblist_dequeue();
 2)               |              file_free_rcu() {
 2)               |                kmem_cache_free() {
 2)   1.792 us    |                  __slab_free();
 2)   5.291 us    |                }
 2)   8.730 us    |              }
 2)   1.646 us    |              rcu_cblist_dequeue();
 2)               |              file_free_rcu() {
 2)               |                kmem_cache_free() {
 2)   1.563 us    |                  __slab_free();
 2)   4.792 us    |                }
 2)   8.000 us    |              }
 2)   1.625 us    |              rcu_cblist_dequeue();
 2)   1.541 us    |              rcu_segcblist_insert_done_cbs();
 2)   1.625 us    |              rcu_segcblist_insert_count();
 2)   1.542 us    |              rcu_segcblist_ready_cbs();
 2)   1.792 us    |              note_gp_changes();
 2)   1.834 us    |              rcu_jiffies_till_stall_check();
 2)   1.583 us    |              rcu_segcblist_ready_cbs();
 2)   1.812 us    |              note_gp_changes();
 2)   1.563 us    |              rcu_jiffies_till_stall_check();
 2)   1.750 us    |              rcu_segcblist_ready_cbs();
 2) ! 111.730 us  |            }
 2)   1.646 us    |            rcu_bh_qs();
 2)               |            irqtime_account_irq() {
 2)   1.562 us    |              irqtime_account_delta.isra.0();
 2)   4.938 us    |            }
 2) ! 153.896 us  |          }
 2)   1.958 us    |          idle_cpu();
 2)               |          rcu_irq_exit() {
 2)               |            rcu_nmi_exit() {
 2)   1.771 us    |              rcu_dynticks_curr_cpu_in_eqs();
 2)   5.270 us    |            }
 2)   8.812 us    |          }
 2) ! 183.125 us  |        }
 2) ! 580.958 us  |      }
 2) # 1038.459 us |    }
 2)   <========== |
 2)               |    preempt_schedule_irq() {
 2)               |      rcu_note_context_switch() {
 2)   1.521 us    |        rcu_sched_qs();
 2)   1.604 us    |        rcu_preempt_qs();
 2)   8.250 us    |      }
 2)   1.646 us    |      _raw_spin_lock();
 2)               |      pick_next_task_fair() {
 2)               |        update_curr() {
 2)   1.562 us    |          __calc_delta();
 2)   1.625 us    |          update_min_vruntime();
 2)   8.375 us    |        }
 2)               |        pick_next_entity() {
 2)   1.667 us    |          wakeup_preempt_entity.isra.9();
 2)   1.563 us    |          clear_buddies();
 2)   8.292 us    |        }
 2)               |        put_prev_entity() {
 2)               |          update_curr() {
 2)   1.604 us    |            update_min_vruntime();
 2)               |            cpuacct_charge() {
 2)   1.854 us    |              __rcu_read_lock();
 2)   1.583 us    |              __rcu_read_unlock();
 2)   8.459 us    |            }
 2)   1.562 us    |            __rcu_read_lock();
 2)               |            __cgroup_account_cputime() {
 2)   1.542 us    |              cgroup_base_stat_cputime_account_begin();
 2)               |              cgroup_base_stat_cputime_account_end.isra.0() {
 2)   1.625 us    |                cgroup_rstat_updated();
 2)   4.854 us    |              }
 2) + 11.625 us   |            }
 2)   1.542 us    |            __rcu_read_unlock();
 2) + 35.000 us   |          }
 2)   1.562 us    |          __enqueue_entity();
 2)   1.625 us    |          __update_load_avg_se();
 2)   1.562 us    |          __update_load_avg_cfs_rq();
 2) + 48.125 us   |        }
 2)               |        put_prev_entity() {
 2)   1.604 us    |          update_curr();
 2)   1.750 us    |          __enqueue_entity();
 2)   1.625 us    |          __update_load_avg_se();
 2)   1.562 us    |          __update_load_avg_cfs_rq();
 2) + 14.854 us   |        }
 2)               |        set_next_entity() {
 2)   1.625 us    |          __update_load_avg_se();
 2)   1.563 us    |          __update_load_avg_cfs_rq();
 2)   8.855 us    |        }
 2) + 99.333 us   |      }
 2)               |      fpsimd_thread_switch() {
 2)   1.708 us    |        fpsimd_save();
 2)   5.062 us    |      }
 2)   1.645 us    |      hw_breakpoint_thread_switch();
 2)   1.666 us    |      uao_thread_switch();
 2)               |      finish_task_switch() {
 2)   1.917 us    |        _raw_spin_unlock_irq();
 2)   5.979 us    |      }
 2) ! 289.521 us  |    }
 2)               |    XXX_gpio_get_direction [gpio_XXX]() {
 2)   1.646 us    |      gpiochip_get_data();
 2)   6.062 us    |    }
 2) # 1351.855 us |  }



// 实验3:设置
/sys/class/gpio/gpio480# echo in > direction


// 重新回到原来的窗口查看结果:
 1)               |  gpiod_direction_input() {
 1)   ==========> |
 1)               |    gic_handle_irq() {
 1)               |      handle_IPI() {
 1)               |        irq_enter() {
 1)               |          rcu_irq_enter() {
 1)               |            rcu_nmi_enter() {
 1)   1.729 us    |              rcu_dynticks_curr_cpu_in_eqs();
 1)   5.458 us    |            }
 1)   8.875 us    |          }
 1)   1.875 us    |          irqtime_account_irq();
 1) + 15.834 us   |        }
 1)               |        __wake_up() {
 1)               |          __wake_up_common_lock() {
 1)   1.688 us    |            _raw_spin_lock_irqsave();
 1)               |            __wake_up_common() {
 1)               |              autoremove_wake_function() {
 1)               |                default_wake_function() {
 1)               |                  try_to_wake_up() {
 1)   1.688 us    |                    _raw_spin_lock_irqsave();
 1)               |                    select_task_rq_fair() {
 1)   1.605 us    |                      __rcu_read_lock();
 1)   1.625 us    |                      available_idle_cpu();
 1)   2.604 us    |                      update_cfs_rq_h_load();
 1)               |                      select_idle_sibling() {
 1)   1.562 us    |                        available_idle_cpu();
 1)   4.750 us    |                      }
 1)   1.563 us    |                      __rcu_read_unlock();
 1) + 22.604 us   |                    }
 1)               |                    smp_send_reschedule() {
 1)               |                      smp_cross_call() {
 1)   1.875 us    |                        gic_raise_softirq();
 1)   5.083 us    |                      }
 1)   8.250 us    |                    }
 1)   1.667 us    |                    _raw_spin_unlock_irqrestore();
 1) + 43.208 us   |                  }
 1) + 46.396 us   |                }
 1) + 49.583 us   |              }
 1) + 53.250 us   |            }
 1)   1.667 us    |            _raw_spin_unlock_irqrestore();
 1) + 63.250 us   |          }
 1) + 66.605 us   |        }
 1)               |        irq_exit() {
 1)               |          irqtime_account_irq() {
 1)   1.584 us    |            irqtime_account_delta.isra.0();
 1)   5.167 us    |          }
 1)   1.812 us    |          idle_cpu();
 1)               |          rcu_irq_exit() {
 1)               |            rcu_nmi_exit() {
 1)   1.562 us    |              rcu_dynticks_curr_cpu_in_eqs();
 1)   4.833 us    |            }
 1)   8.104 us    |          }
 1) + 21.937 us   |        }
 1) ! 112.271 us  |      }
 1) ! 116.208 us  |    }
 1)   <========== |
 1)   1.875 us    |    validate_desc();
 1)               |    XXX_gpio_direction_input [gpio_XXX]() {	// XXX代表具体厂家的芯片驱动前缀,下同
 1)   1.583 us    |      gpiochip_get_data();
 1)   1.729 us    |      _raw_spin_lock_irqsave();
 1)   1.813 us    |      _raw_spin_unlock_irqrestore();
 1) + 12.292 us   |    }
 1) ! 147.104 us  |  }


// 实验4:再回到命令行窗口测试out
/sys/class/gpio/gpio480# echo out > direction

// 再查看ftrace输出窗口
/sys/kernel/debug/tracing# cat trace_pipe 
 0)               |  gpiod_direction_output_raw() {
 0)   ==========> |
 0)               |    gic_handle_irq() {
 0)               |      handle_IPI() {
 0)               |        irq_enter() {
 0)               |          rcu_irq_enter() {
 0)               |            rcu_nmi_enter() {
 0)   1.605 us    |              rcu_dynticks_curr_cpu_in_eqs();
 0)   5.709 us    |            }
 0)   9.021 us    |          }
 0)   1.834 us    |          irqtime_account_irq();
 0) + 15.979 us   |        }
 0)               |        __wake_up() {
 0)               |          __wake_up_common_lock() {
 0)   1.791 us    |            _raw_spin_lock_irqsave();
 0)               |            __wake_up_common() {
 0)               |              autoremove_wake_function() {
 0)               |                default_wake_function() {
 0)               |                  try_to_wake_up() {
 0)   1.687 us    |                    _raw_spin_lock_irqsave();
 0)               |                    select_task_rq_fair() {
 0)   1.625 us    |                      __rcu_read_lock();
 0)   1.563 us    |                      available_idle_cpu();
 0)   2.500 us    |                      update_cfs_rq_h_load();
 0)               |                      select_idle_sibling() {
 0)   1.625 us    |                        available_idle_cpu();
 0)   4.854 us    |                      }
 0)   1.583 us    |                      __rcu_read_unlock();
 0) + 22.583 us   |                    }
 0)               |                    smp_send_reschedule() {
 0)               |                      smp_cross_call() {
 0)   1.916 us    |                        gic_raise_softirq();
 0)   5.042 us    |                      }
 0)   8.229 us    |                    }
 0)   1.667 us    |                    _raw_spin_unlock_irqrestore();
 0) + 43.229 us   |                  }
 0) + 46.395 us   |                }
 0) + 49.584 us   |              }
 0) + 53.646 us   |            }
 0)   1.687 us    |            _raw_spin_unlock_irqrestore();
 0) + 63.791 us   |          }
 0) + 67.166 us   |        }
 0)               |        irq_exit() {
 0)               |          irqtime_account_irq() {
 0)   1.562 us    |            irqtime_account_delta.isra.0();
 0)   4.958 us    |          }
 0)   1.833 us    |          idle_cpu();
 0)               |          rcu_irq_exit() {
 0)               |            rcu_nmi_exit() {
 0)   1.604 us    |              rcu_dynticks_curr_cpu_in_eqs();
 0)   4.833 us    |            }
 0)   8.084 us    |          }
 0) + 21.541 us   |        }
 0) ! 113.083 us  |      }
 0) ! 117.230 us  |    }
 0)   <========== |
 0)   1.833 us    |    validate_desc();
 0)               |    gpiod_direction_output_raw_commit() {
 0)               |      XXX_gpio_direction_output [gpio_XXX]() {
 0)   1.521 us    |        gpiochip_get_data();
 0)   1.771 us    |        _raw_spin_lock_irqsave();
 0)   1.687 us    |        _raw_spin_unlock_irqrestore();
 0)               |        XXX_gpio_set [gpio_XXX]() {
 0)   1.583 us    |          gpiochip_get_data();
 0)   1.813 us    |          _raw_spin_lock_irqsave();
 0)   1.729 us    |          _raw_spin_unlock_irqrestore();
 0) + 12.292 us   |        }
 0) + 26.208 us   |      }
 0) + 30.083 us   |    }
 0) ! 165.021 us  |  }

通过上述代码进一步分析,修改direction的函数是direction_restore, 显示direction的函数是direction_show

参考
关于ftrace,下面这篇文章写得比较好
https://www.ibm.com/developerworks/cn/linux/l-cn-ftrace2/

linux知识库
http://lib.csdn.net/base/linux

linux那些事作者的blog
http://blog.csdn.net/fudan_abc

内核Documentation/trace/ftrace.txt 里面很详细,作为基本参考,包括如何加载ftrace到sys文件系统初始化

阿加
随笔分类 - ftrace
https://www.cnblogs.com/openix/category/641946.html

linux ftrace追踪一(基本技术结构粗略剖析)
https://blog.csdn.net/u011013137/article/details/9093823

【linux内核调试】内核调试方法对比
https://blog.csdn.net/panhewu9919/article/details/103114321/

  • 0
    点赞
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值