Linux性能调试工具之ftrace

1 SA8155 ftrace
1.1 signal知识
shell $?返回值大于128时，表示该脚本收到了signal。

1.2 ftrace-cmd
make LDFLAGS=-static \
CC=aarch64-linux-gnu-gcc trace-cmd

1.3 kill ftrace
1.3.1 enable
echo nop > \
/sys/kernel/debug/tracing/current_tracer
echo syscalls:sys_enter_kill >> \
/sys/kernel/debug/tracing/set_event
echo syscalls:sys_exit_kill >> \
/sys/kernel/debug/tracing/set_event
echo 1 > \
/sys/kernel/debug/tracing/tracing_on

cat /sys/kernel/debug/tracing/trace_pipe

1.3.2 disable
echo 0 > \
/sys/kernel/debug/tracing/tracing_on
echo 0 > \
/sys/kernel/debug/tracing/events/syscalls/sys_exit_kill/enable
echo 0 > \
/sys/kernel/debug/tracing/events/syscalls/sys_enter_kill/enable
echo nop > \
/sys/kernel/debug/tracing/current_tracer

1.4 my_func ftrace
echo 0 > \
/sys/kernel/debug/tracing/tracing_on
echo 0 > \
/sys/kernel/debug/tracing/trace
echo function_graph > \
/sys/kernel/debug/tracing/current_tracer
echo funcgraph-proc > \
/sys/kernel/debug/tracing/trace_options
echo funcgraph-abstime > \
/sys/kernel/debug/tracing/trace_options

echo "my_func1 my_func2" > \
/sys/kernel/debug/tracing/set_ftrace_filter
echo 1 > \
/sys/kernel/debug/tracing/tracing_on

echo 0 > /sys/kernel/debug/tracing/tracing_on
cat /sys/kernel/debug/tracing/trace > \
my_trace_result

1.5 sched_switch
1.5.1 sched_switch
insmod driver, then run ftrace script, this sequence could ensure no latency issue according to my observation.
sched_migrate_task: /proc/sys/kernel/sched_nr_migrate
sched_switch: __schedule()
sched_wakeup: try_to_wake_up()
sched_wakeup_new: for new forked task

# per cpu buffer
echo 10000 > \
/sys/kernel/debug/tracing/buffer_size_kb
# echo "(prev_pid == 1 || next_pid == 1)" > \
# /sys/kernel/debug/tracing/events/sched/sched_switch/filter

# echo 1 > \
# /sys/kernel/debug/tracing/events/irq/irq_handler_entry/enable
# echo 1 > \
# /sys/kernel/debug/tracing/events/irq/irq_handler_exit/enable

echo 1 > \
/sys/kernel/debug/tracing/events/sched/sched_switch/enable
echo 1 > \
/sys/kernel/debug/tracing/events/sched/sched_wakeup_new/enable

cat /sys/kernel/debug/tracing/set_event

echo 1 > \
/sys/kernel/debug/tracing/tracing_on
cat /sys/kernel/debug/tracing/trace

1.5.2 scheduling latency time_diff.py
# cur_line: [CPU number], timestamp
import io
import re
import sys,os
import shutil

arg0_proc_name = ''
def print_usage():
    print('\nUsage: python ' + arg0_proc_name +
        ' <file> <match_string>')

def find_string(src_file,
        match_string):
    last_line_cnt = 0
    cur_line_cnt = 0
    last_ts = 0
    cur_ts = 0
    last_string = ''
    num_of_intr = 0

    fp = io.open(src_file, 'r',
        encoding='utf-8',
        errors='ignore')
    for cur_string in fp:
        cur_line_cnt += 1
        if re.findall(match_string, cur_string):
            list = cur_string.split()
            tmp = float(list[3].rstrip(':'))
            cur_ts = int(tmp * 1000)
            delta = abs(cur_ts - last_ts)
            new_match_string = "next_comm=" +\
                match_string
            if (delta > 5 and
                    re.findall(new_match_string,
                        cur_string)):
                print('++++ delta = ' +
                    str(delta) + 'ms, ' +
                    'num_of_intr = ' + str(num_of_intr))
                print(str(last_line_cnt) + ': ' +
                    last_string)
                print(str(cur_line_cnt) + ': ' +
                    cur_string)

            last_ts = cur_ts
            last_line_cnt = cur_line_cnt
            last_string = cur_string
            num_of_intr = 0
        elif re.findall("irq_handler_exit", cur_string):
            num_of_intr = num_of_intr + 1
    fp.close()

def main():
    global arg0_proc_name
    arg0_proc_name = sys.argv[0]
    if sys.argv[0].rfind(os.path.sep) > 0 :
        index = sys.argv[0].rfind(os.path.sep)
        arg0_proc_name = sys.argv[0][index+1:]

    if len(sys.argv) < 3:
        print_usage()
        sys.exit(0)
    find_string(sys.argv[1], sys.argv[2])

if __name__ == '__main__':
    main()

1.5.3 disable cpuidle
/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
i=0
while [ $i -le 7 ]; do
    echo 1 > /sys/devices/system/cpu/cpu$i/cpuidle/state0/disable
    echo 1 > /sys/devices/system/cpu/cpu$i/cpuidle/state1/disable
    i=$((i + 1))
done

1.6 perfetto
perfetto: English Perfect
stress-ng: a next generation tool designed to stress test system hardware

https://ui.perfetto.dev/#!/viewer

2 htop
ncurses_5.6
./configure \
--host=x86_64-unknown-linux-gnu \
--without-trace \
--without-cxx \
--without-cxx-binding \
--without-ada \
--without-manpages \
--without-progs \
--without-tests \
--with-shared

htop
./configure --disable-unicode \
--host=aarch64-poky-linux

3 ltrace
3.1 Android
https://github.com/LineageOS/android_external_ltrace

3.2 ltrace with Linaro build
3.2.1 zlib
http://www.zlib.net/fossils/

./configure \
--prefix=/path/to/zlib-1.2.11/aarch64

# in Makefile change gcc to
# aarch64-linux-gnu-gcc
make
make install

3.2.2 elfutils
https://sourceware.org/elfutils/ftp/0.178/

LDFLAGS='-L/path/to/zlib-1.2.11/aarch64/lib' \
./configure \
--disable-debuginfod \
--host=aarch64-linux-gnu \
--prefix=/path/to/elfutils-0.178/aarch64

make \
CFLAGS+=\
'-I/path/to/zlib-1.2.11/aarch64/include' \
LDFLAGS+=\
'-Wl,-rpath-link,/path/to/zlib-1.2.11/aarch64/lib' \
LDFLAGS+=\
'-L/path/to/zlib-1.2.11/aarch64/lib'
make install

3.2.3 ltrace
https://github.com/dkogan/ltrace
Do not download ltrace 0.7.3 from http://www.ltrace.org, it is too old, does not support aarch64.
./autogen.sh

CFLAGS=\
'-I/path/to/elfutils-0.178/aarch64/include' \
LDFLAGS=\
'-Wl,-rpath-link,/path/to/zlib-1.2.11/aarch64/lib -L/path/to/elfutils-0.178/aarch64/lib' \
./configure \
--host=aarch64-linux-gnu

[10-Jan-2022] add -ldw, -lelf, -lbz2, -ldl, -llzma, -lpthread for Android 8.0 x86_64.
make \
CFLAGS+=\
'-I/path/to/elfutils-0.178/aarch64/include' \
LDFLAGS+=\
'-L/path/to/elfutils-0.178/aarch64/lib  -ldw -lelf' \
CFLAGS+=\
'-I/path/to/zlib-1.2.11/aarch64/include' \
LDFLAGS+=\
'-L/path/to/zlib-1.2.11/aarch64/lib -lz' \
LDFLAGS+='-lbz2' \
LDFLAGS+='-ldl' \
LDFLAGS+='-llzma' \
LDFLAGS+='-lpthread'

3.2.4 static compilation of ltrace
add -all-static to LINK for libtool in Makefile, shall add it before -o $@, as shown below.
LINK=xxx \
-all-static -o $@

4 strace
4.1 aarch64-linux
./configure --enable-mpers=no \
--host=aarch64-linux

4.2 strace
strace -e epoll_wait -x -p $PID
strace -p $PID -tt -T -s 128 -x 2>&1 | \
grep "read(17"

1) sync time to PC
date -s "2020-02-02 16:02:15"
2) only capture SIGALRM
strace -e signal=SIGALRM \
-e trace=none -tt -p <$PID>
3) capture ioctl
strace -e signal=none \
-e trace=ioctl -tt -p <$PID>
4) capture all strace log
strace -tt -p <$PID>

4.3 How To Enable Userspace Dump Stack
prctl(PR_SET_DUMPABLE, 1);
add this code to rild.

需要添加的sepolicy权限如下：
allow xxx self:capability dac_override;

5 systemd
5.1 code
PulseAudio, systemd suite: by Lennart Poettering
https://github.com/systemd/systemd/releases
systemd-244/src/network/networkd.c

5.2 systemd-networkd
systemd-networkd
high priority: /etc/systemd/network
mid priority: /run/systemd/network
low priority: /usr/lib/systemd/network

systemd-networkd-wait-online
/lib/systemd/system

/lib/systemd/system/systemd-networkd.service
[Service]
ExecStartPre=+/bin/sh /usr/bin/xx.sh
Environment=SYSTEMD_LOG_LEVEL=debug
journalctl -b -u systemd-networkd > \
/tmp/networkd.log

5.3 systemd-analyze
opkg install systemd-analyze.ipk
opkg remove systemd-analyze
ipk: Itsy Package

systemd-analyze blame
systemd-analyze critical-chain \
xxx.service
@后面的时刻表示该单元的启动时刻；+后面的时长表示该单元总计花了多长时间才完成启动。
systemd-analyze plot > boot.svg
systemctl list-dependencies xxx.service

6 Abbreviations
OPP: Operating Performance Points
RBCPR: RapidBridge Core Power Reduction