DebugFs & Kernel Tracer 之 kernel tracer -- mark for not finished

 
1.  ftrace通过debugfs想userspace提供访问接口,编译时内核会使用编译器的-pg选项(该选项在内核的Makefile:
    kernel/trace/Makefile中定义)

2.  配置内核激活debugfs后(CONFIG_DEBUG_FS=y),会创建目录"/sys/kernel/debug".debugfs就是挂载到该目录.
    debugfs不是默认挂载到/sys/kernel/debug目录的，可以通过修改/etc/fstab进行挂载 或 运行时用命令挂载.

    修改fstab将debugfs挂载到/sys/kernel/debug目录的方法: 
        debugfs /sys/kernel/debug debugfs 0 0
    运行时将debugfs挂载到/sys/kernel/debug目录的命令:
        mount -t debugfs nodev /sys/kernel/debug

3.  激活内核对ftracce的支持后,会在debugfs下创建一个tracing目录(例如: /sys/kernel/debug/tracing)
    该目录下包含了ftrace的控制和输出文件.
    根据编译内核时针对ftrace的设定不同,该目录下实际显示的文件和目录也会有所不同.

         

@kernel/trace/trace.h
/**
 * struct tracer - a specific tracer and its callbacks to interact with debugfs
 * @name: the name chosen to select it on the available_tracers file
 * @init: called when one switches to this tracer (echo name > current_tracer)
 * @reset: called when one switches to another tracer
 * @start: called when tracing is unpaused (echo 1 > tracing_enabled)
 * @stop: called when tracing is paused (echo 0 > tracing_enabled)
 * @open: called when the trace file is opened
 * @pipe_open: called when the trace_pipe file is opened
 * @wait_pipe: override how the user waits for traces on trace_pipe
 * @close: called when the trace file is released
 * @pipe_close: called when the trace_pipe file is released
 * @read: override the default read callback on trace_pipe
 * @splice_read: override the default splice_read callback on trace_pipe
 * @selftest: selftest to run on boot (see trace_selftest.c)
 * @print_headers: override the first lines that describe your columns
 * @print_line: callback that prints a trace
 * @set_flag: signals one of your private flags changed (trace_options file)
 * @flags: your private flags
 */

struct tracer {
        const char              *name;
        int                     (*init)(struct trace_array *tr);
        void                    (*reset)(struct trace_array *tr);
        void                    (*start)(struct trace_array *tr);
        void                    (*stop)(struct trace_array *tr);
        void                    (*open)(struct trace_iterator *iter);
        void                    (*pipe_open)(struct trace_iterator *iter);
        void                    (*wait_pipe)(struct trace_iterator *iter);
        void                    (*close)(struct trace_iterator *iter);
        void                    (*pipe_close)(struct trace_iterator *iter);
        ssize_t                 (*read)(struct trace_iterator *iter,
                                        struct file *filp, char __user *ubuf,
                                        size_t cnt, loff_t *ppos);
        ssize_t                 (*splice_read)(struct trace_iterator *iter,
                                               struct file *filp,
                                               loff_t *ppos,
                                               struct pipe_inode_info *pipe,
                                               size_t len,
                                               unsigned int flags);
#ifdef CONFIG_FTRACE_STARTUP_TEST
        int                     (*selftest)(struct tracer *trace,
                                            struct trace_array *tr);
#endif
        void                    (*print_header)(struct seq_file *m);
        enum print_line_t       (*print_line)(struct trace_iterator *iter);
        /* If you handled the flag setting, return 0 */
        int                     (*set_flag)(u32 old_flags, u32 bit, int set);
        struct tracer           *next;
        struct tracer_flags     *flags;
        int                     print_max;
        int                     use_max_tr;
};

//trace_type是一个由tracer结构构成的单链表的表头
//static struct tracer       *trace_types __read_mostly;
//register_tracer(struct tracer *type)就是将tracer注册到trace_types链表中去
int register_tracer(struct tracer *type)
+-- struct tracer *t;
+-- mutex_lock(&trace_types_lock);
+-- tracing_selftest_running = true;
//如果tracer已经在trace_types链表中,则说明已经注册,直接退出.
+-- for (t = trace_types; t; t = t->next) {
        if (strcmp(type->name, t->name) == 0) {
            /* already found */
            pr_info("Tracer %s already registered\n", type->name);
            ret = -1;
            goto out;
        }
    }
+-- if (!type->set_flag)
           type->set_flag = &dummy_set_flag;
+-- if (!type->flags)
           type->flags = &dummy_tracer_flags;
    else
        if (!type->flags->opts)
            type->flags->opts = dummy_tracer_opt;
+-- if (!type->wait_pipe)
            type->wait_pipe = default_wait_pipe;
//将type链接到trace_types链表中
+-- type->next = trace_types;
+-- trace_types = type;
//out:
+-- tracing_selftest_running = false;
+-- mutex_unlock(&trace_types_lock);
+-- printk(KERN_INFO "Starting tracer '%s'\n", type->name);
+-- tracing_set_tracer(type->name);
+-- default_bootup_tracer = NULL;
+-- tracing_selftest_disabled = 1;

//ftrace就是通过tracer_init()创建的debugfs文件来和用户空间进行交互的,
//具体动作有xxx_fops结构定义的接口函数集完成.
//fs_initcall(tracer_init_debugfs);
static __init int tracer_init_debugfs(void)
+-- struct dentry *d_tracer;
+-- trace_access_lock_init();
+-- d_tracer = tracing_init_dentry();
//在debugfs中创建各种文件接口作为向user space提供的控制和数据IO接口,接口函数实现为一些列的文件操作接口函数集
//(struct file_operations),诸如:tracing_ctrl_fops,tracing_iter_fops等等.
+-- trace_create_file("tracing_enabled", 0644, d_tracer, &global_trace, &tracing_ctrl_fops);             
+-- trace_create_file("trace_options", 0644, d_tracer, NULL, &tracing_iter_fops);                       
+-- trace_create_file("tracing_cpumask",0644, d_tracer, NULL, &tracing_cpumask_fops);                    
+-- trace_create_file("trace", 0644, d_tracer,(void *) TRACE_PIPE_ALL_CPU, &tracing_fops);      
+-- trace_create_file("available_tracers",0444, d_tracer, &global_trace, &show_traces_fops);               
+-- trace_create_file("current_tracer", 0644, d_tracer, &global_trace, &set_tracer_fops);                
+-- trace_create_file("tracing_max_latency", 0644, d_tracer, &tracing_max_latency, &tracing_max_lat_fops);    
+-- trace_create_file("tracing_thresh", 0644, d_tracer, &tracing_thresh, &tracing_max_lat_fops);         
+-- trace_create_file("README",0444, d_tracer, NULL, &tracing_readme_fops);                     
+-- trace_create_file("trace_pipe", 0444, d_tracer, (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); 
+-- trace_create_file("buffer_size_kb", 0644, d_tracer, &global_trace, &tracing_entries_fops);           
+-- trace_create_file("buffer_total_size_kb", 0444, d_tracer, &global_trace, &tracing_total_entries_fops);     
+-- trace_create_file("free_buffer", 0644, d_tracer, &global_trace, &tracing_free_buffer_fops);      
+-- trace_create_file("trace_marker", 0220, d_tracer, NULL, &tracing_mark_fops);                      
+-- trace_create_file("saved_cmdlines", 0444, d_tracer, NULL, &tracing_saved_cmdlines_fops);            
+-- trace_create_file("trace_clock", 0644, d_tracer, NULL, &trace_clock_fops);                       
+-- trace_create_file("tracing_on", 0644, d_tracer, &global_trace, &rb_simple_fops);                
+-- trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, &ftrace_update_tot_cnt, &tracing_dyn_info_fops);

+-- create_trace_options_dir();

+-- for_each_tracing_cpu(cpu)
       tracing_init_debugfs_percpu(cpu); 


struct dentry *trace_create_file(const char *name, umode_t mode, struct dentry *parent,
                                 void *data, const struct file_operations *fops)
+-- struct dentry *ret;
+-- ret = debugfs_create_file(name, mode, parent, data, fops);

//early_initcall(tracer_alloc_buffers);
__init static int 
tracer_alloc_buffers(void)

+-- alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)
+-- alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)

+-- if (ring_buffer_expanded)
        ring_buf_size = trace_buf_size;
    else
        ring_buf_size = 1;

+-- rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;

+-- cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
+-- cpumask_copy(tracing_cpumask, cpu_all_mask);

+-- global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
+-- global_trace.entries = ring_buffer_size(global_trace.buffer);

+-- max_tr.buffer = ring_buffer_alloc(1, rb_flags);
+-- max_tr.entries = 1;

    //Allocate the first page for all buffers 
+-- for_each_tracing_cpu(i) {
        global_trace.data[i] = &per_cpu(global_trace_cpu, i);
        max_tr.data[i] = &per_cpu(max_tr_data, i);
    }

+-- trace_init_cmdlines();
+-- register_tracer(&nop_trace);
+-- current_trace = &nop_trace;

    //All seems OK, enable tracing
+-- tracing_disabled = 0;
 
+-- atomic_notifier_chain_register(&panic_notifier_list, &trace_panic_notifier);

+-- register_die_notifier(&trace_die_notifier);

//各种tracer可以在各个阶段的初始化过程中调用register_tracer()函数进行注册.
//kernel/trace/trace_sched_wakeup.c
//device_initcall(init_wakeup_tracer);
__init static int init_wakeup_tracer(void)
+-- ret = register_tracer(&wakeup_tracer);      //wakeup_tracer ... 
+-- ret = register_tracer(&wakeup_rt_tracer);   //wakeup_rt_tracer ...

static struct tracer wakeup_tracer __read_mostly =
{
        .name           = "wakeup",
        .init           = wakeup_tracer_init,
        ...
};

static struct tracer wakeup_rt_tracer __read_mostly =
{
        .name           = "wakeup_rt",
        .init           = wakeup_rt_tracer_init,
        ...
};

一. ftrace 的跟踪器

    ftrace 当前包含多个跟踪器，用于跟踪不同类型的信息，比如进程调度、中断关闭等。可以查看文件 available_tracers 获取内核当前支持的跟踪器列表。在编译内核时，也可以看到内核支持的跟踪器对应的选项。

nop跟踪器
nop跟踪器不会跟踪任何内核活动，将 nop 写入 current_tracer 文件可以删除之前所使用的跟踪器，并清空之前收集到的跟踪信息，即刷新 trace 文件。 

function跟踪器
function跟踪器可以跟踪内核函数的执行情况;
可以通过文件 set_ftrace_filter 显示指定要跟踪的函数。 

function_graph跟踪器
function_graph跟踪器可以显示类似 C 源码的函数调用关系图，这样查看起来比较直观一些;
可以通过文件 set_grapch_function 显示指定要生成调用流程图的函数. 

sched_switch跟踪器
sched_switch跟踪器可以对内核中的进程调度活动进行跟踪。 

irqsoff跟踪器和 preemptoff跟踪器
这两个跟踪器分别跟踪关闭中断的代码和禁止进程抢占的代码，并记录关闭的最大时长.

preemptirqsoff跟踪器
preemptirqsoff跟踪器可以看做irqsoff跟踪器和 preemptoff跟踪器的组合.
 
    ftrace 还支持其它一些跟踪器，比如 initcall, ksym_tracer, mmiotrace, sysprof 等.
ftrace 框架支持扩展添加新的跟踪器.可以参考内核源码包中 Documentation/trace 目录下的文档以及 kernel/trace 下的源文件, 以了解其它跟踪器的用途和如何添加新的跟踪器.

二. ftrace 的数据文件

    /sys/kernel/debug/trace 目录下文件和目录比较多，有些是各种跟踪器共享使用的，有些是特定于某个跟踪器使用的.在操作这些数据文件时，通常使用 echo 命令来修改其值，也可以在程序中通过文件读写相关的函数来操作这些文件的值.可以参考内核源码包中 Documentation/trace 目录下的文档以及 kernel/trace 下的源文件以了解各个文件的用途.

README文件: 
提供了一个简短的使用说明，展示了 ftrace 的操作命令序列。
可以通过 cat 命令查看该文件以了解概要的操作流程。 

current_tracer
用于设置或显示当前使用的跟踪器；
使用 echo 将跟踪器名字写入该文件可以切换到不同的跟踪器。
系统启动后，其缺省值为 nop, 即不做任何跟踪操作。
在执行完一段跟踪任务后, 可以通过向该文件写入 nop 来重置跟踪器. 

available_tracers
记录了当前编译进内核的跟踪器的列表.
可以通过 cat 查看其内容;
其包含的跟踪器与Make menuconfig配置Tracer选项时所激活的选项是对应的.
写 current_tracer 文件时用到的跟踪器名字必须在该文件列出的跟踪器名字列表中. 

trace
文件提供了查看获取到的跟踪信息的接口.
可以通过 cat 等命令查看该文件以查看跟踪到的内核活动记录，也可以将其内容保存为记录文件以备后续查看。 

tracing_enabled
用于控制 current_tracer 中的跟踪器是否可以跟踪内核函数的调用情况.
写入 0 会关闭跟踪活动, 写入 1 则激活跟踪功能;
其缺省值为 1 .  

set_graph_function
设置要清晰显示调用关系的函数，显示的信息结构类似于 C 语言代码，这样在分析内核运作流程时会更加直观一些.
在使用 function_graph 跟踪器时使用;
缺省为对所有函数都生成调用关系序列, 可以通过写该文件来指定需要特别关注的函数. 

buffer_size_kb
用于设置单个 CPU 所使用的跟踪缓存的大小.
跟踪器会将跟踪到的信息写入缓存，每个 CPU 的跟踪缓存是一样大的.
跟踪缓存实现为环形缓冲区的形式，如果跟踪到的信息太多，则旧的信息会被新的跟踪信息覆盖掉.
注意，要更改该文件的值需要先将 current_tracer 设置为 nop 才可以。 

tracing_on
用于控制跟踪的暂停.
有时候在观察到某些事件时想暂时关闭跟踪,可以将 0 写入该文件以停止跟踪,
这样跟踪缓冲区中比较新的部分是与所关注的事件相关的; 写入 1 可以继续跟踪. 

available_filter_functions
记录了当前可以跟踪的内核函数.
对于不在该文件中列出的函数,无法跟踪其活动. 

set_ftrace_filter 和 set_ftrace_notrace
在编译内核时配置了动态 ftrace (选中 CONFIG_DYNAMIC_FTRACE 选项)后使用.
前者用于显示指定要跟踪的函数, 后者则作用相反, 用于指定不跟踪的函数.
如果一个函数名同时出现在这两个文件中，则这个函数的执行状况不会被跟踪.
这些文件还支持简单形式的含有通配符的表达式, 这样可以用一个表达式一次指定多个目标函数.
注意, 要写入这两个文件的函数名必须可以在文件 available_filter_functions 中看到. 
缺省为可以跟踪所有内核函数，文件 set_ftrace_notrace 的值则为空. 

1. tracing_enabled
tracing_enabled用于控制 current_tracer 中的跟踪器是否可以跟踪内核函数的调用情况。
写入 0 会关闭跟踪活动,写入 1 则激活跟踪功能;
其缺省值为 1.

static __init int 
tracer_init_debugfs(void)
+-- trace_create_file("tracing_enabled", 0644, d_tracer, &global_trace, &tracing_ctrl_fops);
    +-- debugfs_create_file(name, mode, parent, data, fops);

static const struct file_operations tracing_ctrl_fops = {
        .open           = tracing_open_generic,
        .read           = tracing_ctrl_read,
        .write          = tracing_ctrl_write,
        .llseek         = generic_file_llseek,
};

static ssize_t
tracing_ctrl_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+-- struct trace_array *tr = filp->private_data;
+-- ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
+-- mutex_lock(&trace_types_lock);
+-- if (tracer_enabled ^ val) {
        if (val) {
            tracer_enabled = 1;
            if (current_trace->start)
                current_trace->start(tr); //调用tracer的start()成员
            tracing_start();              //操作ringbuffer相关的东西...
        } else {
            tracer_enabled = 0;
            tracing_stop();               //操作ringbuffer相关的东西...
            if (current_trace->stop)
                current_trace->stop(tr);  //调用tracer的stop()成员
        }
    }
+-- mutex_unlock(&trace_types_lock);
+-- *ppos += cnt;

static ssize_t
tracing_ctrl_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
+-- r = sprintf(buf, "%u\n", tracer_enabled);
+-- simple_read_from_buffer(ubuf, cnt, ppos, buf, r);


void tracing_start(void)
+-- if (tracing_disabled)
        return;
+-- buffer = global_trace.buffer;
+-- if (buffer)
        ring_buffer_record_enable(buffer);
+-- buffer = max_tr.buffer;
+-- if (buffer)
        ring_buffer_record_enable(buffer);
+-- ftrace_start();
    +-- function_trace_stop = 0;


void tracing_stop(void)
+-- ftrace_stop();
    +-- function_trace_stop = 1;
+-- trace_stop_count++;
+-- buffer = global_trace.buffer;
+-- if (buffer)
        ring_buffer_record_disable(buffer);
+-- buffer = max_tr.buffer;
+-- if (buffer)
        ring_buffer_record_disable(buffer);

2. current_tracer
current_tracer用于设置或显示当前使用的tracer(跟踪器); 
使用 echo 将跟踪器名字写入该文件可以切换到不同的tracer(跟踪器)。
系统启动后,其缺省值为 nop,即不做任何跟踪操作.在执行完一段跟踪任务后,可以通过向该文件写入 nop 来重置跟踪器.

static __init int   
tracer_init_debugfs(void) 
+-- trace_create_file("current_tracer", 0644, d_tracer, &global_trace, &set_tracer_fops);
    +-- debugfs_create_file(name, mode, parent, data, fops);

static const struct file_operations set_tracer_fops = {
        .open           = tracing_open_generic,
        .read           = tracing_set_trace_read,
        .write          = tracing_set_trace_write,
        .llseek         = generic_file_llseek,
};

static ssize_t
tracing_set_trace_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
+-- copy_from_user(&buf, ubuf, cnt)
+-- tracing_set_tracer(buf);
    +-- struct trace_array *tr = &global_trace;     //global_tracer...
    +-- __tracing_resize_ring_buffer(trace_buf_size);
        //根据buf指定的tracer的name,在trace_types链表里面找到已经注册的tracer.
    +-- for (t = trace_types; t; t = t->next) {
             if (strcmp(t->name, buf) == 0)
                 break;
        }
    +-- trace_branch_disable();
    +-- if (current_trace && current_trace->reset)
            current_trace->reset(tr);   //调用current_tracer的reset()
    +-- if (current_trace && current_trace->use_max_tr) {
            //We don't free the ring buffer. instead, resize it because The max_tr ring buffer
            //has some state (e.g. ring->clock) and we want preserve it.
            ring_buffer_resize(max_tr.buffer, 1);
            max_tr.entries = 1;
        }
    +-- destroy_trace_option_files(topts);
    +-- current_trace = t;     //将current_trace设置为ubuf字符串对应的struct tracer结构.
    +-- topts = create_trace_option_files(current_trace);

    +-- if (current_trace->use_max_tr) {
            ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
            if (ret < 0)
                goto out;
            max_tr.entries = global_trace.entries;
        }  
    +-- if (t->init) {
                ret = tracer_init(t, tr); //调用新找到的tracer的init()
                +-- tracing_reset_online_cpus(tr);
                +-- t->init(tr);
        } 
    +-- trace_branch_enable(tr);

3. trace
trace文件提供了查看获取到的跟踪信息的接口。
可以通过 cat 等命令查看该文件以查看跟踪到的内核活动记录,也可以将其内容保存为记录文件以备后续查看。 

static __init int   
tracer_init_debugfs(void) 
+-- trace_create_file("trace", 0644, d_tracer, (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
    +-- debugfs_create_file(name, mode, parent, data, fops); 

static const struct file_operations tracing_fops = {
        .open           = tracing_open,         //tracing_open...
        .read           = seq_read,             //seq_read...
        .write          = tracing_write_stub,
        .llseek         = tracing_seek,
        .release        = tracing_release,
};

static int 
tracing_open(struct inode *inode, struct file *file)
+-- if (file->f_mode & FMODE_READ) 
        iter = __tracing_open(inode, file);
    
static struct trace_iterator *
__tracing_open(struct inode *inode, struct file *file)
+-- long cpu_file = (long) inode->i_private;
+-- struct trace_iterator *iter;
+-- struct seq_file *m;

+-- if (tracing_disabled)
        return ERR_PTR(-ENODEV); 
+-- iter = kzalloc(sizeof(*iter), GFP_KERNEL);   //创建并初始化一个trace_iterator结构. 
+-- iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);

+-- if (current_trace) *iter->trace = *current_trace;
+-- zalloc_cpumask_var(&iter->started, GFP_KERNEL)
+-- if (current_trace && current_trace->print_max) 
        iter->tr = &max_tr;
    else 
        iter->tr = &global_trace;
+-- mutex_init(&iter->mutex);
+-- iter->cpu_file = cpu_file;

+-- f (iter->trace && iter->trace->open) iter->trace->open(iter); //调用tracer的open()
+-- tracing_stop();

+-- ..
+-- cpu = iter->cpu_file;
+-- iter->buffer_iter[cpu] = ring_buffer_read_prepare(iter->tr->buffer, cpu);
+-- ring_buffer_read_prepare_sync();
+-- ring_buffer_read_start(iter->buffer_iter[cpu]);
+-- tracing_iter_reset(iter, cpu);

+-- ret = seq_open(file, &tracer_seq_ops);
    +-- struct seq_file *p = file->private_data;
    +-- memset(p, 0, sizeof(*p));
    +-- p->op = op;   //seq_file的op指针成员就是在此被初始化成了&tracer_seq_ops. seq_read()函数里会用到

+-- m = file->private_data;
    //iter被创建并初始化之后被作为seq_file的private保存下来,此后通过seq_file的private就可以拿到iter.
    //进而拿到tracer.
+-- m->private = iter; 
+-- return iter;

//trace_iterator的定义如下:
//Trace iterator - used by printout routines who present trace
//results to users and which routines might sleep, etc:
struct trace_iterator {
        struct trace_array      *tr;
        struct tracer           *trace;
        void                    *private;
        int                     cpu_file;
        struct mutex            mutex;
        struct ring_buffer_iter *buffer_iter[NR_CPUS];
        unsigned long           iter_flags;
        //trace_seq for __print_flags() and __print_symbolic() etc.
        struct trace_seq        tmp_seq;
        //The below is zeroed out in pipe_read
        struct trace_seq        seq;
        struct trace_entry      *ent;
        ...
};


ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos)
+-- struct seq_file *m = file->private_data;
+-- pos = m->index;
+-- p = m->op->start(m, &pos);            //实际上就是tracer_seq_ops.start()
+-- while (1) {
        err = m->op->show(m, p);          //实际上就是tracer_seq_ops.show()
        if (unlikely(!m->count)) {
            p = m->op->next(m, p, &pos);  //实际上就是tracer_seq_ops.next()
            m->index = pos;
            continue;
        }
        m->op->stop(m, p);                //实际上就是tracer_seq_ops.stop()
        kfree(m->buf);
        m->buf = kmalloc(m->size <<= 1, GFP_KERNEL);
        m->count = 0;
        m->version = 0;
        pos = m->index;
        p = m->op->start(m, &pos);        //实际上就是tracer_seq_ops.start()
    }
+-- m->op->stop(m, p);
+-- m->count = 0;
+-- ...
+-- m->op->stop(m, p);
+-- n = min(m->count, size);
+-- err = copy_to_user(buf, m->buf, n);   //将获得的m->buf中的内容拷贝到user space中的buf里面去
+-- ...



static const struct seq_operations tracer_seq_ops = {
        .start          = s_start,
        .next           = s_next,
        .stop           = s_stop,
        .show           = s_show,
};


static int s_show(struct seq_file *m, void *v)
{
        struct trace_iterator *iter = v;
        int ret;

        if (iter->ent == NULL) {
                if (iter->tr) {
                        seq_printf(m, "# tracer: %s\n", iter->trace->name);
                        seq_puts(m, "#\n");
                        test_ftrace_alive(m);
                }
                if (iter->trace && iter->trace->print_header)
                        iter->trace->print_header(m);
                else
                        trace_default_header(m);

        } else if (iter->leftover) {
                //If we filled the seq_file buffer earlier, we
                //want to just show it now.
                ret = trace_print_seq(m, &iter->seq);   //(iter->seq).buffer => (m->buf + m->count)
                //ret should this time be zero, but you never know
                iter->leftover = ret;

        } else {
                print_trace_line(iter);                        //将消息放进(iter->seq).buffer
                ret = trace_print_seq(m, &iter->seq);   //(iter->seq).buffer => (m->buf + m->count)
                //If we overflow the seq_file buffer, then it will
                //ask us for this data again at start up.
                //Use that instead.
                //ret is 0 if seq_file write succeeded.
                //      -1 otherwise.
                iter->leftover = ret;
        }

        return 0;
}
 
//将s->buffer中的数据拷贝到(m->buf + m->count)中去
int trace_print_seq(struct seq_file *m, struct trace_seq *s)
+-- ret = seq_write(m, s->buffer, len);                 //s->buffer => (m->buf+m->count)
          +-- memcpy(seq->buf + seq->count, data, len); //data => (seq->buf+seq->count)
          +-- seq->count += len;
          +-- return 0;
    +-- if (!ret)
            trace_seq_init(s);


//那么(iter->seq).buffer中的数据又是哪来的呢？,由print_trace_line(iter);中来:
//看下这个函数的实现:
enum print_line_t print_trace_line(struct trace_iterator *iter)
{
        enum print_line_t ret;

        if (iter->lost_events &&
            !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
                                 iter->cpu, iter->lost_events))
                return TRACE_TYPE_PARTIAL_LINE;

        if (iter->trace && iter->trace->print_line) {
                ret = iter->trace->print_line(iter);
                if (ret != TRACE_TYPE_UNHANDLED)
                        return ret;
        }

        if (iter->ent->type == TRACE_BPRINT &&
                        trace_flags & TRACE_ITER_PRINTK &&
                        trace_flags & TRACE_ITER_PRINTK_MSGONLY)
                return trace_print_bprintk_msg_only(iter);

        if (iter->ent->type == TRACE_PRINT &&
                        trace_flags & TRACE_ITER_PRINTK &&
                        trace_flags & TRACE_ITER_PRINTK_MSGONLY)
                return trace_print_printk_msg_only(iter);

        if (trace_flags & TRACE_ITER_BIN)
                return print_bin_fmt(iter);

        if (trace_flags & TRACE_ITER_HEX)
                return print_hex_fmt(iter);

        if (trace_flags & TRACE_ITER_RAW)
                return print_raw_fmt(iter);

        return print_trace_fmt(iter);
}


enum print_line_t 
print_trace_line(struct trace_iterator *iter)
?-- trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", iter->cpu, iter->lost_events))
    +-- vsnprintf(s->buffer + s->len, len, fmt, ap); //s->buffer即为(&iter->seq)->buffer
?-- iter->trace->print_line(iter);
?-- trace_print_bprintk_msg_only(iter);
    +-- trace_seq_bprintf(s, field->fmt, field->buf);
        +-- vsnprintf(s->buffer + s->len, len, fmt, ap); //s->buffer即为(&iter->seq)->buffer
?-- print_bin_fmt(iter);
    +-- entry = iter->ent;
    +-- event = ftrace_find_event(entry->type);
    +-- event->funcs->binary(iter, 0, event)
?-- print_hex_fmt(iter);
    +-- entry = iter->ent;
    +-- event = ftrace_find_event(entry->type);
    +-- event->funcs->hex(iter, 0, event);
?-- print_raw_fmt(iter);
    +-- entry = iter->ent;
    +-- event = ftrace_find_event(entry->type);
    +-- event->funcs->raw(iter, 0, event);
?-- print_trace_fmt(iter);
    +-- entry = iter->ent;
    +-- event = ftrace_find_event(entry->type);
    +-- event->funcs->trace(iter, sym_flags, event);

下面的问题是找到tracer被加到trace_types链表中之后，是如何被使用的?