关键词:fcgi、environ、getenv、段错误、segment fault、FCGI_Accept
fastcgi 更改环境变量environ引起的段错误(一)fcgi库代码分析
fastcgi 更改环境变量environ引起的段错误(二)修改libfcgi库源码解决getenv段错误问题
fastcgi 更改环境变量environ引起的段错误(三)getenv函数的使用
fastcgi在收到HTTP服务器(例如 nginx)新请求后,FCGI_Accept为全局变量stdin、stdout、stderr和environ 赋新值,即改变存放环境变量的全局变量 char **environ ,如果进程的线程使用 C库函数getenv()读取环境变量全局变量 char **environ会引起段错误。
在fcgi 进程中使用 mosquitto 创建客户端连接 mosquitto 订阅mqtt消息,如果需要重启 /usr/bin/mosquitto,这时fcgi 进程的"mosquitto loop" 线程会重新连接mosquitto,并调用函数getaddrinfo()->getenv(),由于fcgi的函数FCGI_Accept接收来自HTTP服务器的新请求时会修改存放环境变量的全局变量 char **environ ,如果同进程的线程"mosquitto loop"使用 C库函数getenv()读取环境变量全局变量 char **environ会引起段错误。
fastcgi在接收HTTP服务器(例如 nginx)请求前会先将上次创建的 static FCGX_Request the_request 清除, 其中在函数FreeParams(&request->paramsPtr) 中会执行 free(&request->paramsPtr->vec) ,也就是把全局变量 char **environ给 free了,这时再调用函数getenv()获取的 __environ 处存放的数据是非法地址,会引起段错误。
FCGI_Accept()先执行 FCGX_Finish_r(reqDataPtr)->FCGX_Free(reqDataPtr, close)->free(paramsPtr->vec) 先清除处理上次请求的数据,其中 paramsPtr->vec 赋值给全局变量 char **environ,在执行 free(paramsPtr->vec) 后,函数getenv()获取的全局变量 char **environ 指向的内存地址被free了,这时再使用getenv 再使用 environ 就会出现段错误。
接收到新请求后会在如下的调用过程中使用 malloc 分配堆空间,并让 全局变量 char **environ 指向这个堆空间:
FCGX_Finish_r(reqDataPtr)-> //先清除处理上次请求的数据
close |= FCGX_FClose(reqDataPtr->err);
close |= FCGX_FClose(reqDataPtr->out);
FCGX_Free(reqDataPtr, close)->
FreeParams(&request->paramsPtr)->
free(paramsPtr->vec); //paramsPtr->vec 赋值给全局变量 char **environ,free 之后再执行 getenv()将会出现段错误
free(paramsPtr);
reqDataPtr->paramsPtr = NewParams(30)->
result = (Params *)Malloc(sizeof(Params));
result->vec = (char **)Malloc(length * sizeof(char *));//paramsPtr->vec 赋值给全局变量 char **environ
在下面这个while循环体里范围内因为函数FCGI_Accept()已经对全局变量 char **environ地址进行了重新赋值,使用 getenv()是不会产生段错误的:
while (FCGI_Accept() >= 0) { //函数FCGI_Accept 将修改了全局变量 char **environ地址
cp = getenv("LOCALDOMAIN"); //在while循环体里执行 getenv() 没有问题,不会发生段错误
}
mqtt线程重新连接mosquitto 服务器的过程如下:
mosquitto-2.0.10/lib/thread_mosq.c
int mosquitto_loop_start(struct mosquitto *mosq)->
pthread_create(&mosq->thread_id, NULL, mosquitto__thread_main, mosq);
pthread_setname_np(mosq->thread_id, "mosquitto loop");
void *mosquitto__thread_main(void *obj)->
mosquitto_loop_forever(mosq, mosq->keepalive*1000, 1)->
while(run)
{
rc = mosquitto_loop(mosq, timeout, max_packets);
do{
rc = mosquitto_reconnect(mosq)->
return mosquitto__reconnect(mosq, true)->
net__socket_connect(mosq, mosq->host, mosq->port, mosq->bind_address, blocking)->
rc = net__try_connect(host, port, &mosq->sock, bind_address, blocking)->
return net__try_connect_tcp(host, port, sock, bind_address, blocking);
s = getaddrinfo(host, NULL, &hints, &ainfo)->
gaih_inet.constprop->
__resolv_context_get->
maybe_init->
__res_vinit->
__resolv_conf_load->
getenv("LOCALDOMAIN") //产生段错误
}
while(run && rc != MOSQ_ERR_SUCCESS);
}
产生段错误时gdb调试信息如下,所使用的 libc.so 使用 gcc -O2编译优化选项,getenv()出现段错误的汇编代码如下:
ldr r4, [r5] # 前面已将变量 __environ 的地址赋值给寄存器 r5,#fcgi的函数FCGI_Accept 将全局变量 char **environ地址修改为0xdc1ba0
61 if (name_start == ep_start && !strncmp (*ep + 2, name, len)
0x00010600 <+104>: ldrh r3, [r4] #出现段错误的地方 #将存储器地址为R4的半字数据读入寄存器R3,并将R3的高16位清零。
# 对应 uint16_t ep_start = (((unsigned char *) *ep)[0]
(gdb) c
Continuing.
Thread 2 "mosquitto loop" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0xb5e31390 (LWP 597)]
0xb687bf90 in getenv () from /data/mxlApp/libs/libc.so.6
(gdb) bt
#0 0xb687bf90 in getenv () from /data/mxlApp/libs/libc.so.6
#1 0xb693e150 in __resolv_conf_load () from /data/mxlApp/libs/libc.so.6
#2 0xb69404ec in __resolv_conf_get_current () from /data/mxlApp/libs/libc.so.6
#3 0xb693ec98 in __res_vinit () from /data/mxlApp/libs/libc.so.6
#4 0xb693fc90 in maybe_init () from /data/mxlApp/libs/libc.so.6
#5 0xb693fe10 in __resolv_context_get () from /data/mxlApp/libs/libc.so.6
#6 0xb69073bc in gaih_inet.constprop () from /data/mxlApp/libs/libc.so.6
#7 0xb6908574 in getaddrinfo () from /data/mxlApp/libs/libc.so.6
#8 0xb6d3fc2c in net.try_connect () from /data/mxlApp/libs/libmosquitto.so.1
#9 0xb6d3fdd4 in net.socket_connect () from /data/mxlApp/libs/libmosquitto.so.1
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) info thread
Id Target Id Frame
1 Thread 0xb6fe8010 (LWP 587) "mxl_cgi" 0xb6e817d4 in accept () from /lib/libpthread.so.0
* 2 Thread 0xb5e31390 (LWP 597) "mosquitto loop" 0xb687bf90 in getenv () from /data/mxlApp/libs/libc.so.6
3 Thread 0xb6632390 (LWP 608) "mxl_cgi" 0xb68e4ad4 in __clock_nanosleep_time64 () from /data/mxlApp/libs/libc.so.6
(gdb) layout
Undefined command: "layout". Try "help".
(gdb) info registers
r0 0xb 11
r1 0xb69761d8 3063374296
r2 0x18 24
r3 0xff000000 4278190080
r4 0xdc1 3521 #此时的 r4 存放的是 *environ ,也就是第一个环境变量的地址,显然 0xdc1 是一个非法地址
#处理完一次请求,FCGI_Accept()在执行 free(paramsPtr->vec) 后,
#函数getenv()获取的全局变量 char **environ 指向的内存地址被free了
r5 0xdc1ba0 14424992 #fcgi的函数FCGI_Accept 将全局变量 char **environ地址修改为0xdc1ba0
r6 0x4f4c 20300
r7 0xb 11
r8 0x9 9
r9 0xb69761da 3063374298
r10 0x5 5
r11 0x2 2
r12 0xffffffff 4294967295
sp 0xb5e301d8 0xb5e301d8
lr 0xb687bf6c -1232617620
pc 0xb687bf90 0xb687bf90 <getenv+112>
cpsr 0x200f0010 537853968
fpscr 0x60000010 1610612752
模仿fcgi和mosquitto以及getenv出现段错误
将下面的代码保存在文件 mxl_cgi.c 中:
#define _GNU_SOURCE
#include <stdlib.h>
#include "fcgi_stdio.h"
#include <stdbool.h>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <unistd.h>
FILE *fp;
void* pthread_run1(void* arg)
{
char **ep;
char *cp;
while(1)
{
//setenv("mxl", "linfen", 1);
//lt_setenv("mxl", "linfen");
fprintf(fp,"I am thread 1,ID: 0x%lx.\n",pthread_self());
fprintf(fp,"__environ = %p.\n", __environ);
//处理完一次请求,FCGI_Accept()在执行 free(paramsPtr->vec) 后,
//函数getenv()获取的全局变量 char **environ 指向的内存地址被free了
//函数FCGI_Accept 将修改了全局变量 char **environ地址,处理完一次请求,FCGI_Accept()在执行 free(paramsPtr->vec) 后,导到这里的 getenv出现段错误
cp = getenv("LOCALDOMAIN");
if( cp == NULL)
perror("get LOCALDOMAIN error:");
else
fprintf(fp, "env LOCALDOMAIN = %s.\n", cp);
for (ep = __environ; *ep != NULL; ++ep) //在这里执行会段错误
{
fprintf(fp,"address of ep= %p , ep = %s\n", ep, *ep);
}
fprintf(fp,"====================================================\n");
fflush(fp);
sleep(5);
}
}
FILE *fp;
int main(int iArgc, char *pcArgv[])
{
pthread_t tid1;
struct sigaction sa;
char **ep;
sa.sa_handler = SIG_IGN;
sa.sa_flags = 0;
if (sigemptyset(&sa.sa_mask) == -1 || //初始化信号集为空
sigaction(SIGPIPE, &sa, 0) == -1)
{
perror("failed to ignore SIGPIPE; sigaction");
exit(EXIT_FAILURE);
}
fp = fopen("/data/fcgi.log", "wb"); //将日志写放文件 /data/fcgi.log
fprintf(fp,"#########################\n");
pthread_create(&tid1,NULL, pthread_run1,NULL);
#if 1
/* 接收前端请求 */
while (FCGI_Accept() >= 0) {//函数FCGI_Accept 将修改了全局变量 char **environ地址
fprintf(fp,"FCGI Accept \n");
fprintf(fp,"REQUEST_METHOD = %s.\n", secure_getenv("REQUEST_METHOD"));
fprintf(fp,"DOCUMENT_URI = %s.\n", secure_getenv("DOCUMENT_URI"));
fprintf(fp,"CONTENT_TYPE = %s.\n", secure_getenv("CONTENT_TYPE"));
fprintf(fp,"SERVER_ADDR = %s.\n", secure_getenv("SERVER_ADDR"));
fprintf(fp,"QUERY_STRING = %s.\n", secure_getenv("QUERY_STRING"));
fprintf(fp,"main thread : __environ = %p.\n", __environ);
for (ep = __environ; *ep != NULL; ++ep)
{
fprintf(fp,"main thread address of ep= %p , ep = %s\n", ep, *ep);
}
fprintf(fp,"********************************************\n");
fflush(fp);
}
#else
sleep(10);
while (1) {
sleep(1);
}
#endif
fclose(fp);
return 0;
}/*End of Main*/
gdb调试过程
将文件 mxl_cgi.c 编译后运行,进程mxl_cgi启动后在谷歌浏览器向cgi 发送 post 请求,进程mxl_cgi就会在 getenv()处发生段错误。
fcgi的函数FCGI_Accept()接收到请求后会修改全局变量 __environ 的值:
全局变量__envrion的初始化参见:
环境变量可以在 命令行中 设置和创建,但用户 退出o命令时 这些 变量值就会丢失 ,因此,如果希望永久保存环境变量,可在用户家目录下的 .bash_profile 或 .bashrc (非用户登录模式特有,例如远程SSH)文件中,或者全 局配置 /etc/bashrc (非用户登录模式特有,例如远程 SSH )或 /etc/profile文件 中定义,在将环境变量放入上述的文件中后,每次用户登录时这些变量都将被初始化。
按照系统规范,所有环境变量的名字均采用大写形式 ,在将环境变量应用于用户进程程序之前,都应该用 export命令导出定义,例如:正确的环境变量定义方法为 export OLDGIRL=1
假设有可执行程序 hello_world, 在 bash shell 中使用 ./hello_world 或者 /bin/bash hello_world 运行可执行文件时,进程 hello_world 会继承bash shell 的环境变量。
在 bash-5.0 为例说明 bash shell 初始化环境变量 environ 的过程 :
int main (argc, argv, env) //bash-5.0/shell.c
int argc;
char **argv, **env;
env = environ; //存放环境变量的全局变量environ
check_dev_tty ();
set_default_locale ();
shell_environment = env;
set_shell_name (argv[0]);
arg_index = parse_long_options (argv, arg_index, argc);
set_login_shell ("login_shell", login_shell != 0);
shell_initialize ()-> //Initialize internal and environment variables. //初始化环境变量
initialize_shell_variables (shell_environment, privileged_mode||running_setuid)-> //初始化全局变量environ
create_variable_tables ();
for (string_index = 0; env && (string = env[string_index++]); )
{
string_length = strlen (string);
temp_string = (char *)xmalloc (namelen + string_length + 2);
memcpy (temp_string, tname, namelen);
temp_string[namelen] = ' ';
memcpy (temp_string + namelen + 1, string, string_length + 1);
}
run_by_ssh = (find_variable ("SSH_CLIENT") != (SHELL_VAR *)0) ||
(find_variable ("SSH2_CLIENT") != (SHELL_VAR *)0);
maybe_execute_file (bashrc_file, 1)-> //根据文件 /etc/profile 设置环境变量: #define SYS_PROFILE "/etc/profile"
rval = _evalfile (filename, flags)->
fd = open (filename, O_RDONLY);
maybe_execute_file ("~/.profile", 1);
在 arm32 的 RK3128 上, gdb调试过程如下:
root@mxlos:/data# gdb /usr/bin/spawn-fcgi
GNU gdb (GDB) 9.2
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "arm-linux-gnueabihf".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /usr/bin/spawn-fcgi...
(No debugging symbols found in /usr/bin/spawn-fcgi)
(gdb) set args -n -a 127.0.0.1 -p 8288 -f /data/mxlApp/bin/mxl_cgi #设置 spawn-fcgi 的运行参数
(gdb) set env LD_LIBRARY_PATH=/data/mxlApp/libs #设置进程运行的环境变量
(gdb) run #开始执行
Starting program: /usr/bin/spawn-fcgi -n -a 127.0.0.1 -p 8288 -f /data/mxlApp/bin/mxl_cgi
process 18658 is executing new program: /bin/bash
process 18658 is executing new program: /userdata/mxlApp/bin/mxl_cgi
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/libthread_db.so.1".
[New Thread 0xb6dc83e0 (LWP 18689)]
Thread 2 "mxl_cgi" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0xb6dc83e0 (LWP 18689)]
0xb6e60f90 in getenv () from /lib/libc.so.6
(gdb) backtrace #查看栈回溯
#0 0xb6e60f90 in getenv () from /lib/libc.so.6
#1 0x00010d3c in pthread_run1 (arg=<optimized out>) at /home/mxl/tmp/wifimesh_app/gateway/mxl_cgi/web_server_app.c:39
#2 0xb6f95df8 in start_thread () from /lib/libpthread.so.0
#3 0xb6f04ae8 in ?? () from /lib/libc.so.6
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) info thread #查看线程信息
Id Target Id Frame
1 Thread 0xb6ff7010 (LWP 18658) "mxl_cgi" 0xb6fa17d4 in accept () from /lib/libpthread.so.0
* 2 Thread 0xb6dc83e0 (LWP 18689) "mxl_cgi" 0xb6e60f90 in getenv () from /lib/libc.so.6 #出现段错误的是线程 2
(gdb) info frame #查看栈帧
Stack level 0, frame at 0xb6dc7d58:
pc = 0xb6e60f90 in getenv; saved pc = 0x10d3c
called by frame at 0xb6dc7d70
Arglist at 0xb6dc7d38, args:
Locals at 0xb6dc7d38, Previous frame's sp is 0xb6dc7d58
Saved registers:
r4 at 0xb6dc7d38, r5 at 0xb6dc7d3c, r6 at 0xb6dc7d40, r7 at 0xb6dc7d44, r8 at 0xb6dc7d48, r9 at 0xb6dc7d4c, r10 at 0xb6dc7d50, lr at 0xb6dc7d54
(gdb)
Stack level 0, frame at 0xb6dc7d58:
pc = 0xb6e60f90 in getenv; saved pc = 0x10d3c
called by frame at 0xb6dc7d70
Arglist at 0xb6dc7d38, args:
Locals at 0xb6dc7d38, Previous frame's sp is 0xb6dc7d58
Saved registers:
r4 at 0xb6dc7d38, r5 at 0xb6dc7d3c, r6 at 0xb6dc7d40, r7 at 0xb6dc7d44, r8 at 0xb6dc7d48, r9 at 0xb6dc7d4c, r10 at 0xb6dc7d50, lr at 0xb6dc7d54
(gdb) frame 0 #查看栈帧0
#0 0xb6e60f90 in getenv () from /lib/libc.so.6
(gdb) frame 1 #查看栈帧1
#1 0x00010d3c in pthread_run1 (arg=<optimized out>) at /home/mxl/tmp/wifimesh_app/gateway/mxl_cgi/web_server_app.c:39
warning: Source file is more recent than executable.
39 cp = getenv("LOCALDOMAIN");
(gdb) frame 2 #查看栈帧2
#2 0xb6f95df8 in start_thread () from /lib/libpthread.so.0
(gdb) frame 3 #查看栈帧3
#3 0xb6f04ae8 in ?? () from /lib/libc.so.6
(gdb) frame 4
No frame at level 4.
(gdb)
(gdb)
(gdb) info registers #查看出错时的寄存器值
r0 0xb 11
r1 0x10ee0 69344
r2 0x8 8
r3 0xff00 65280
r4 0x26 38 #r4 存放的是getenv函数中变量 char **ep 的值,在执行 0xb6e60f90 <+112>: ldrh r3, [r4] 时出现段错误,因为 r4=0x26是非法地址
r5 0x265c0 157120 #寄存器r5存放的环境变量全局变量extern char ** environ 的值,在 FCGI_Accept 接收HTTP服务器(例如 nginx)时将environ修改为0x643ba0
#main函数入口刚执行时 __environ = 0xbefffd3c,地址 0xbefffd3c在进程的栈空间范围[stack]
#处理完一次请求,FCGI_Accept()在执行 free(paramsPtr->vec) 后,
#函数getenv()获取的全局变量 char **environ 指向的内存地址被free了
r6 0x4f4c 20300 #此时r6=0x4F4C, 其中 0x4F对应ascii码的字母O,0x4C 对应ascii码的字母L,即getenv("LOCALDOMAIN") 的入口参数的头两个字符
r7 0xb 11
r8 0x9 9
r9 0x10ee8 69352
r10 0xb6dc83e0 3067904992
r11 0xb6dc7e9c 3067903644
r12 0xffffffff 4294967295
sp 0xb6dc7d38 0xb6dc7d38 #栈指针寄存器
lr 0xb6e60f6c -1226436756
pc 0xb6e60f90 0xb6e60f90 <getenv+112>
cpsr 0x200f0010 537853968
fpscr 0x0 0
(gdb) p **__environ
$1 = 83 'S'
(gdb) p **(__environ+1)
$2 = 69 'E'
(gdb) p **(__environ+2)
$3 = 80 'P'
(gdb) p **(__environ+3)
$4 = 76 'L'
(gdb) p **(__environ+4)
$5 = 95 '_'
(gdb) p **(__environ+5)
$6 = 76 'L'
(gdb) p **(__environ+6)
$7 = 72 'H'
(gdb) info locals
No symbol table info available.
(gdb) info args
No symbol table info available.
(gdb)
(gdb) disassemble #查看出错地方的汇编代码,符号 => 指向的就是出错的位置
Dump of assembler code for function getenv:
0xb6e60f20 <+0>: ldr r3, [pc, #228] ; 0xb6e6100c <getenv+236>
0xb6e60f24 <+4>: ldr r2, [pc, #228] ; 0xb6e61010 <getenv+240>
0xb6e60f28 <+8>: add r3, pc, r3
0xb6e60f2c <+12>: push {r4, r5, r6, r7, r8, r9, r10, lr}
0xb6e60f30 <+16>: ldr r2, [r3, r2]
0xb6e60f34 <+20>: ldr r5, [r2]
0xb6e60f38 <+24>: cmp r5, #0
0xb6e60f3c <+28>: beq 0xb6e61004 <getenv+228>
0xb6e60f40 <+32>: ldrb r2, [r0]
0xb6e60f44 <+36>: mov r6, r0
0xb6e60f48 <+40>: cmp r2, #0
0xb6e60f4c <+44>: moveq r4, r2
0xb6e60f50 <+48>: beq 0xb6e60fc8 <getenv+168>
0xb6e60f54 <+52>: ldrb r3, [r0, #1]
0xb6e60f58 <+56>: ldr r4, [r5]
0xb6e60f5c <+60>: cmp r3, #0
0xb6e60f60 <+64>: beq 0xb6e60fd0 <getenv+176>
0xb6e60f64 <+68>: add r9, r6, #2
0xb6e60f68 <+72>: blx 0xb6ea85c0 <strlen>
0xb6e60f6c <+76>: cmp r4, #0
0xb6e60f70 <+80>: mov r7, r0
0xb6e60f74 <+84>: sub r8, r0, #2
0xb6e60f78 <+88>: beq 0xb6e60fc8 <getenv+168>
0xb6e60f7c <+92>: ldrh r6, [r6]
0xb6e60f80 <+96>: b 0xb6e60f90 <getenv+112>
0xb6e60f84 <+100>: ldr r4, [r5, #4]!
0xb6e60f88 <+104>: cmp r4, #0
0xb6e60f8c <+108>: beq 0xb6e60fc8 <getenv+168>
=> 0xb6e60f90 <+112>: ldrh r3, [r4] #出现段错误的地方#寄存器 r3 存放的 0xf00 是一个非法地址#将存储器地址为R4的半字数据读入寄存器R3,并将R3的高16位清零。
# 对应 uint16_t ep_start = (((unsigned char *) *ep)[0]
0xb6e60f94 <+116>: cmp r3, r6
0xb6e60f98 <+120>: bne 0xb6e60f84 <getenv+100>
0xb6e60f9c <+124>: add r0, r4, #2
0xb6e60fa0 <+128>: mov r2, r8
0xb6e60fa4 <+132>: mov r1, r9
0xb6e60fa8 <+136>: bl 0xb6ea87ec <strncmp>
0xb6e60fac <+140>: cmp r0, #0
0xb6e60fb0 <+144>: bne 0xb6e60f84 <getenv+100>
0xb6e60fb4 <+148>: ldrb r3, [r4, r7]
0xb6e60fb8 <+152>: cmp r3, #61 ; 0x3d
0xb6e60fbc <+156>: bne 0xb6e60f84 <getenv+100>
0xb6e60fc0 <+160>: add r7, r7, #1
0xb6e60fc4 <+164>: add r4, r4, r7
0xb6e60fc8 <+168>: mov r0, r4
0xb6e60fcc <+172>: pop {r4, r5, r6, r7, r8, r9, r10, pc}
0xb6e60fd0 <+176>: cmp r4, #0
0xb6e60fd4 <+180>: orr r2, r2, #15616 ; 0x3d00
0xb6e60fd8 <+184>: bne 0xb6e60fec <getenv+204>
0xb6e60fdc <+188>: b 0xb6e60fc8 <getenv+168>
0xb6e60fe0 <+192>: ldr r4, [r5, #4]!
0xb6e60fe4 <+196>: cmp r4, #0
0xb6e60fe8 <+200>: beq 0xb6e60fc8 <getenv+168>
0xb6e60fec <+204>: ldrsh r3, [r4]
0xb6e60ff0 <+208>: cmp r3, r2
0xb6e60ff4 <+212>: bne 0xb6e60fe0 <getenv+192>
0xb6e60ff8 <+216>: add r4, r4, #2
0xb6e60ffc <+220>: mov r0, r4
0xb6e61000 <+224>: pop {r4, r5, r6, r7, r8, r9, r10, pc}
0xb6e61004 <+228>: mov r4, r5
0xb6e61008 <+232>: b 0xb6e60fc8 <getenv+168>
0xb6e6100c <+236>: ldrsbeq r3, [r1], -r0
0xb6e61010 <+240>: strdeq r0, [r0], -r12
End of assembler dump.
(gdb) disassemble/s
Dump of assembler code for function getenv:
0xb6e60f20 <+0>: ldr r3, [pc, #228] ; 0xb6e6100c <getenv+236>
0xb6e60f24 <+4>: ldr r2, [pc, #228] ; 0xb6e61010 <getenv+240>
0xb6e60f28 <+8>: add r3, pc, r3
0xb6e60f2c <+12>: push {r4, r5, r6, r7, r8, r9, r10, lr}
0xb6e60f30 <+16>: ldr r2, [r3, r2]
0xb6e60f34 <+20>: ldr r5, [r2]
0xb6e60f38 <+24>: cmp r5, #0
0xb6e60f3c <+28>: beq 0xb6e61004 <getenv+228>
0xb6e60f40 <+32>: ldrb r2, [r0]
0xb6e60f44 <+36>: mov r6, r0
0xb6e60f48 <+40>: cmp r2, #0
0xb6e60f4c <+44>: moveq r4, r2
0xb6e60f50 <+48>: beq 0xb6e60fc8 <getenv+168>
0xb6e60f54 <+52>: ldrb r3, [r0, #1]
0xb6e60f58 <+56>: ldr r4, [r5]
0xb6e60f5c <+60>: cmp r3, #0
0xb6e60f60 <+64>: beq 0xb6e60fd0 <getenv+176>
0xb6e60f64 <+68>: add r9, r6, #2
0xb6e60f68 <+72>: blx 0xb6ea85c0 <strlen>
0xb6e60f6c <+76>: cmp r4, #0
0xb6e60f70 <+80>: mov r7, r0
0xb6e60f74 <+84>: sub r8, r0, #2
0xb6e60f78 <+88>: beq 0xb6e60fc8 <getenv+168>
0xb6e60f7c <+92>: ldrh r6, [r6]
0xb6e60f80 <+96>: b 0xb6e60f90 <getenv+112>
0xb6e60f84 <+100>: ldr r4, [r5, #4]!
0xb6e60f88 <+104>: cmp r4, #0
0xb6e60f8c <+108>: beq 0xb6e60fc8 <getenv+168>
=> 0xb6e60f90 <+112>: ldrh r3, [r4]
0xb6e60f94 <+116>: cmp r3, r6
0xb6e60f98 <+120>: bne 0xb6e60f84 <getenv+100>
0xb6e60f9c <+124>: add r0, r4, #2
0xb6e60fa0 <+128>: mov r2, r8
0xb6e60fa4 <+132>: mov r1, r9
0xb6e60fa8 <+136>: bl 0xb6ea87ec <strncmp>
0xb6e60fac <+140>: cmp r0, #0
0xb6e60fb0 <+144>: bne 0xb6e60f84 <getenv+100>
0xb6e60fb4 <+148>: ldrb r3, [r4, r7]
0xb6e60fb8 <+152>: cmp r3, #61 ; 0x3d
0xb6e60fbc <+156>: bne 0xb6e60f84 <getenv+100>
0xb6e60fc0 <+160>: add r7, r7, #1
0xb6e60fc4 <+164>: add r4, r4, r7
0xb6e60fc8 <+168>: mov r0, r4
0xb6e60fcc <+172>: pop {r4, r5, r6, r7, r8, r9, r10, pc}
0xb6e60fd0 <+176>: cmp r4, #0
0xb6e60fd4 <+180>: orr r2, r2, #15616 ; 0x3d00
0xb6e60fd8 <+184>: bne 0xb6e60fec <getenv+204>
0xb6e60fdc <+188>: b 0xb6e60fc8 <getenv+168>
0xb6e60fe0 <+192>: ldr r4, [r5, #4]!
0xb6e60fe4 <+196>: cmp r4, #0
0xb6e60fe8 <+200>: beq 0xb6e60fc8 <getenv+168>
0xb6e60fec <+204>: ldrsh r3, [r4]
0xb6e60ff0 <+208>: cmp r3, r2
0xb6e60ff4 <+212>: bne 0xb6e60fe0 <getenv+192>
0xb6e60ff8 <+216>: add r4, r4, #2
0xb6e60ffc <+220>: mov r0, r4
0xb6e61000 <+224>: pop {r4, r5, r6, r7, r8, r9, r10, pc}
0xb6e61004 <+228>: mov r4, r5
0xb6e61008 <+232>: b 0xb6e60fc8 <getenv+168>
0xb6e6100c <+236>: ldrsbeq r3, [r1], -r0
0xb6e61010 <+240>: strdeq r0, [r0], -r12
End of assembler dump.
(gdb) shell ps aux | grep cgi #执行 shell 命令查看进程mxl_cgi的 PID 为 18658
root 18632 0.1 1.9 10944 9360 pts/4 S+ 14:22 0:00 gdb /usr/bin/spawn-fcgi
root 18658 0.0 0.0 10712 440 pts/4 tl 14:22 0:00 /data/mxlApp/bin/mxl_cgi
root 21388 0.0 0.4 3168 2112 pts/4 S+ 14:27 0:00 sh -c ps aux | grep cgi
root 21391 0.0 0.0 2796 368 pts/4 S+ 14:27 0:00 grep cgi
(gdb) shell pmap -x 18658 #使用 pmap 查看进程空间
18658: /data/mxlApp/bin/mxl_cgi
Address Kbytes RSS Dirty Mode Mapping
00010000 8 8 4 r-x-- mxl_cgi
00021000 4 4 4 r---- mxl_cgi
00022000 4 4 4 rw--- mxl_cgi
00023000 132 16 16 rw--- [ anon ]
b65c8000 4 0 0 ----- [ anon ]
b65c9000 8192 8 8 rw--- [ anon ]
b6dc9000 352 64 0 r-x-- libm-2.32.so
b6e21000 60 0 0 ----- libm-2.32.so
b6e30000 4 4 4 r---- libm-2.32.so
b6e31000 4 4 4 rw--- libm-2.32.so
b6e32000 1216 952 16 r-x-- libc-2.32.so
b6f62000 64 0 0 ----- libc-2.32.so
b6f72000 8 8 8 r---- libc-2.32.so
b6f74000 4 4 4 rw--- libc-2.32.so
b6f75000 12 8 8 rw--- [ anon ]
b6f78000 28 28 0 r-x-- libfcgi.so.0.0.0
b6f7f000 60 0 0 ----- libfcgi.so.0.0.0
b6f8e000 4 4 4 r---- libfcgi.so.0.0.0
b6f8f000 4 4 4 rw--- libfcgi.so.0.0.0
b6f90000 88 88 8 r-x-- libpthread-2.32.so
b6fa6000 60 0 0 ----- libpthread-2.32.so
b6fb5000 4 4 4 r---- libpthread-2.32.so
b6fb6000 4 4 4 rw--- libpthread-2.32.so
b6fb7000 8 4 4 rw--- [ anon ]
b6fb9000 8 8 0 r-x-- libdl-2.32.so
b6fbb000 60 0 0 ----- libdl-2.32.so
b6fca000 4 4 4 r---- libdl-2.32.so
b6fcb000 4 4 4 rw--- libdl-2.32.so
b6fcc000 140 140 8 r-x-- ld-2.32.so
b6ff7000 16 12 12 rw--- [ anon ]
b6ffb000 4 0 0 r-x-- [ anon ]
b6ffc000 4 0 0 r---- [ anon ]
b6ffd000 4 4 0 r-x-- [ anon ]
b6ffe000 4 4 4 r---- ld-2.32.so
b6fff000 4 4 4 rw--- ld-2.32.so
befdf000 132 8 8 rw--- [ stack ] #进程栈空间范围
ffff0000 4 0 0 r-x-- [ anon ]
-------- ------- ------- -------
total kB 10716 1408 152
(gdb) shell cat /proc/18658/maps ##执行shell命令查看进程空间
00010000-00012000 r-xp 00000000 b3:0a 202 /userdata/mxlApp/bin/mxl_cgi
00021000-00022000 r--p 00001000 b3:0a 202 /userdata/mxlApp/bin/mxl_cgi
00022000-00023000 rw-p 00002000 b3:0a 202 /userdata/mxlApp/bin/mxl_cgi
00023000-00044000 rw-p 00000000 00:00 0 [heap]
b65c8000-b65c9000 ---p 00000000 00:00 0
b65c9000-b6dc9000 rw-p 00000000 00:00 0
b6dc9000-b6e21000 r-xp 00000000 b3:09 321 /lib/libm-2.32.so
b6e21000-b6e30000 ---p 00058000 b3:09 321 /lib/libm-2.32.so
b6e30000-b6e31000 r--p 00057000 b3:09 321 /lib/libm-2.32.so
b6e31000-b6e32000 rw-p 00058000 b3:09 321 /lib/libm-2.32.so
b6e32000-b6f62000 r-xp 00000000 b3:09 311 /lib/libc-2.32.so
b6f62000-b6f72000 ---p 00130000 b3:09 311 /lib/libc-2.32.so
b6f72000-b6f74000 r--p 00130000 b3:09 311 /lib/libc-2.32.so
b6f74000-b6f75000 rw-p 00132000 b3:09 311 /lib/libc-2.32.so
b6f75000-b6f78000 rw-p 00000000 00:00 0
b6f78000-b6f7f000 r-xp 00000000 b3:09 1110 /usr/lib/libfcgi.so.0.0.0
b6f7f000-b6f8e000 ---p 00007000 b3:09 1110 /usr/lib/libfcgi.so.0.0.0
b6f8e000-b6f8f000 r--p 00006000 b3:09 1110 /usr/lib/libfcgi.so.0.0.0
b6f8f000-b6f90000 rw-p 00007000 b3:09 1110 /usr/lib/libfcgi.so.0.0.0
b6f90000-b6fa6000 r-xp 00000000 b3:09 338 /lib/libpthread-2.32.so
b6fa6000-b6fb5000 ---p 00016000 b3:09 338 /lib/libpthread-2.32.so
b6fb5000-b6fb6000 r--p 00015000 b3:09 338 /lib/libpthread-2.32.so
b6fb6000-b6fb7000 rw-p 00016000 b3:09 338 /lib/libpthread-2.32.so
b6fb7000-b6fb9000 rw-p 00000000 00:00 0
b6fb9000-b6fbb000 r-xp 00000000 b3:09 315 /lib/libdl-2.32.so
b6fbb000-b6fca000 ---p 00002000 b3:09 315 /lib/libdl-2.32.so
b6fca000-b6fcb000 r--p 00001000 b3:09 315 /lib/libdl-2.32.so
b6fcb000-b6fcc000 rw-p 00002000 b3:09 315 /lib/libdl-2.32.so
b6fcc000-b6fef000 r-xp 00000000 b3:09 301 /lib/ld-2.32.so
b6ff7000-b6ffb000 rw-p 00000000 00:00 0
b6ffb000-b6ffc000 r-xp 00000000 00:00 0 [sigpage]
b6ffc000-b6ffd000 r--p 00000000 00:00 0 [vvar]
b6ffd000-b6ffe000 r-xp 00000000 00:00 0 [vdso]
b6ffe000-b6fff000 r--p 00022000 b3:09 301 /lib/ld-2.32.so
b6fff000-b7000000 rw-p 00023000 b3:09 301 /lib/ld-2.32.so
befdf000-bf000000 rw-p 00000000 00:00 0 [stack] #进程栈空间范围
ffff0000-ffff1000 r-xp 00000000 00:00 0 [vectors]
(gdb) info registers
r0 0xb 11
r1 0x10ee0 69344
r2 0x8 8
r3 0xff00 65280
r4 0x26 38
r5 0x265c0 157120
r6 0x4f4c 20300
r7 0xb 11
r8 0x9 9
r9 0x10ee8 69352
r10 0xb6dc83e0 3067904992
r11 0xb6dc7e9c 3067903644
r12 0xffffffff 4294967295
sp 0xb6dc7d38 0xb6dc7d38
lr 0xb6e60f6c -1226436756
pc 0xb6e60f90 0xb6e60f90 <getenv+112>
cpsr 0x200f0010 537853968
fpscr 0x0 0
(gdb) x/s $r1
0x10ee0: "%lx.\n"
(gdb) x/s $r5
0x265c0: "&"
(gdb) x/s $r9
0x10ee8: "CALDOMAIN" #存放调用 getenv("LOCALDOMAIN") 时的入口参数
(gdb) x/s $r0
0xb: <error: Cannot access memory at address 0xb>
(gdb) x/16x $sp-16 #查看栈寄存器sp 指向的内存地址附近的数据
0xb6dc7d28: 0xcc 0x0e 0x01 0x00 0x87 0x10 0x01 0x00
0xb6dc7d30: 0x26 0x00 0x00 0x00 0xc0 0x65 0x02 0x00
(gdb) x/16c $sp-16
0xb6dc7d28: -52 '\314' 14 '\016' 1 '\001' 0 '\000' -121 '\207' 16 '\020' 1 '\001' 0 '\000'
0xb6dc7d30: 38 '&' 0 '\000' 0 '\000' 0 '\000' -64 '\300' 101 'e' 2 '\002' 0 '\000'
(gdb)
gdb调试生成的coredump 文件
coredump 配置如下:
ulimit -c 150000
echo 1 > /proc/sys/kernel/core_uses_pid
echo "/data/mxlApp/data/corefile/core-%e-%p-%s-%t" > /proc/sys/kernel/core_pattern
mxl_cgi 产生段错误后生成的 coredump 文件如下:
root@mxlos:/data/mxlApp/data/corefile# ll --full-time
total 180
drwxr-xr-x 2 root root 4096 2023-08-23 15:50:34 +0800 ./
drwxrwxr-x 6 root root 4096 2023-08-22 18:53:50 +0800 ../
-rw------- 1 root root 8806400 2023-08-23 15:50:34 +0800 core-mxl_cgi-3390-11-1692777034
gdb调试生成的coredump 文件过程如下:
root@mxlos:/data/mxlApp/data/corefile# gdb /usr/bin/spawn-fcgi
GNU gdb (GDB) 9.2
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "arm-linux-gnueabihf".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /usr/bin/spawn-fcgi...
(No debugging symbols found in /usr/bin/spawn-fcgi)
(gdb) set args -n -a 127.0.0.1 -p 8288 -f /data/mxlApp/bin/mxl_cgi
(gdb) set env LD_LIBRARY_PATH=/data/mxlApp/libs
(gdb) core-file core-mxl_cgi-3390-11-1692777034
warning: core file may not match specified executable file.
[New LWP 3391]
[New LWP 3390]
Core was generated by `/data/mxlApp/bin/mxl_cgi'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0xb6daaf90 in ?? ()
[Current thread is 1 (LWP 3391)]
(gdb) bt
#0 0xb6daaf90 in ?? () #执行 info proc mappings 查看出错的 0xb6daaf90 处理libc 库的范围,说明出错的是 libc 里的库函数
#1 0xb6daaf6c in ?? ()
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) info registers
r0 0xb 11
r1 0x10ee0 69344
r2 0x8 8
r3 0xff00 65280
r4 0x3dd 989
r5 0x3dd5c0 4052416
r6 0x4f4c 20300
r7 0xb 11
r8 0x9 9
r9 0x10ee8 69352
r10 0xb6d123e0 3067159520
r11 0xb6d11e9c 3067158172
r12 0xffffffff 4294967295
sp 0xb6d11d38 0xb6d11d38
lr 0xb6daaf6c -1227182228
pc 0xb6daaf90 0xb6daaf90
cpsr 0x200f0010 537853968
fpscr 0x0 0
(gdb) info proc mappings
Mapped address spaces:
Start Addr End Addr Size Offset objfile
0x10000 0x12000 0x2000 0x0 /userdata/mxlApp/bin/mxl_cgi
0x21000 0x22000 0x1000 0x1000 /userdata/mxlApp/bin/mxl_cgi
0x22000 0x23000 0x1000 0x2000 /userdata/mxlApp/bin/mxl_cgi
0xb6d13000 0xb6d6b000 0x58000 0x0 /lib/libm-2.32.so
0xb6d6b000 0xb6d7a000 0xf000 0x58000 /lib/libm-2.32.so
0xb6d7a000 0xb6d7b000 0x1000 0x57000 /lib/libm-2.32.so
0xb6d7b000 0xb6d7c000 0x1000 0x58000 /lib/libm-2.32.so
0xb6d7c000 0xb6eac000 0x130000 0x0 /lib/libc-2.32.so #栈回溯出错的 0xb6daaf90 处理libc 库的范围,说明出错的是 libc 里的库函数
0xb6eac000 0xb6ebc000 0x10000 0x130000 /lib/libc-2.32.so
0xb6ebc000 0xb6ebe000 0x2000 0x130000 /lib/libc-2.32.so
0xb6ebe000 0xb6ebf000 0x1000 0x132000 /lib/libc-2.32.so
0xb6ec2000 0xb6ec9000 0x7000 0x0 /usr/lib/libfcgi.so.0.0.0
0xb6ec9000 0xb6ed8000 0xf000 0x7000 /usr/lib/libfcgi.so.0.0.0
0xb6ed8000 0xb6ed9000 0x1000 0x6000 /usr/lib/libfcgi.so.0.0.0
0xb6ed9000 0xb6eda000 0x1000 0x7000 /usr/lib/libfcgi.so.0.0.0
0xb6eda000 0xb6ef0000 0x16000 0x0 /lib/libpthread-2.32.so
0xb6ef0000 0xb6eff000 0xf000 0x16000 /lib/libpthread-2.32.so
0xb6eff000 0xb6f00000 0x1000 0x15000 /lib/libpthread-2.32.so
0xb6f00000 0xb6f01000 0x1000 0x16000 /lib/libpthread-2.32.so
0xb6f03000 0xb6f05000 0x2000 0x0 /lib/libdl-2.32.so
0xb6f05000 0xb6f14000 0xf000 0x2000 /lib/libdl-2.32.so
0xb6f14000 0xb6f15000 0x1000 0x1000 /lib/libdl-2.32.so
0xb6f15000 0xb6f16000 0x1000 0x2000 /lib/libdl-2.32.so
0xb6f16000 0xb6f39000 0x23000 0x0 /lib/ld-2.32.so
0xb6f48000 0xb6f49000 0x1000 0x22000 /lib/ld-2.32.so
#到现在为止可以知道的栈回溯出错的 0xb6daaf90 处理libc 库的范围,说明出错的是 libc 里的库函数,但不知道是哪个函数了错。
#这时可以打印下栈指针寄存器指向的内存地址附近的值
(gdb)
(gdb) x/16x $sp-16
0xb6d11d28: 0xcc 0x0e 0x01 0x00 0x87 0x10 0x01 0x00
0xb6d11d30: 0xdd 0x03 0x00 0x00 0xc0 0xd5 0x3d 0x00
(gdb) x/16c $sp-16
0xb6d11d28: -52 '\314' 14 '\016' 1 '\001' 0 '\000' -121 '\207' 16 '\020' 1 '\001' 0 '\000'
0xb6d11d30: -35 '\335' 3 '\003' 0 '\000' 0 '\000' -64 '\300' -43 '\325' 61 '=' 0 '\000'
(gdb) x/32c $sp-32
0xb6d11d18: 5 '\005' 0 '\000' 0 '\000' 0 '\000' 104 'h' 32 ' ' 2 '\002' 0 '\000'
0xb6d11d20: -32 '\340' 35 '#' -47 '\321' -74 '\266' 96 '`' 32 ' ' 2 '\002' 0 '\000'
0xb6d11d28: -52 '\314' 14 '\016' 1 '\001' 0 '\000' -121 '\207' 16 '\020' 1 '\001' 0 '\000'
0xb6d11d30: -35 '\335' 3 '\003' 0 '\000' 0 '\000' -64 '\300' -43 '\325' 61 '=' 0 '\000'
(gdb) x/32c $sp
0xb6d11d38: 104 'h' 32 ' ' 2 '\002' 0 '\000' -32 '\340' 35 '#' -47 '\321' -74 '\266'
0xb6d11d40: 96 '`' 32 ' ' 2 '\002' 0 '\000' -52 '\314' 14 '\016' 1 '\001' 0 '\000'
0xb6d11d48: -121 '\207' 16 '\020' 1 '\001' 0 '\000' 112 'p' -86 '\252' -25 '\347' -66 '\276'
0xb6d11d50: -32 '\340' 35 '#' -47 '\321' -74 '\266' 60 '<' 13 '\r' 1 '\001' 0 '\000'
(gdb)
如果还是没有有价值信息,那么可以尝试打印出寄存器是否存放有字符串,例如本例中r9存放 "CALDOMAIN",因此可以在libc库或者进程源码中搜索键字"CALDOMAIN"
(gdb) x/s $r9
0x10ee8 <abort@plt+8>: "CALDOMAIN"
(gdb) x/s $sp
0xb6d11d38: "h \002"
(gdb) x/s $pc
0xb6daaf90: <error: Cannot access memory at address 0xb6daaf90>
(gdb) x/s $lr
0xb6daaf6c: <error: Cannot access memory at address 0xb6daaf6c>
到这里还是没有线索时,可以将 not stripped 的 libc库复制到目录 /data/mxlApp/libs,使用set env LD_LIBRARY_PATH=/data/mxlApp/libs 设置库的优先搜索路径
然后再执行 run 执行 mxl_cgi,接着执行 ctrl+C 暂停gdb 的运行,然后再使用 core-file core-mxl_cgi-3390-11-1692777034 加载 coredump 文件,这样再执行 backtrace 时可以看到出错的函数是 getenv :
root@mxlos:/data/mxlApp/data/corefile# gdb /usr/bin/spawn-fcgi
GNU gdb (GDB) 9.2
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "arm-linux-gnueabihf".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /usr/bin/spawn-fcgi...
(No debugging symbols found in /usr/bin/spawn-fcgi)
(gdb) set args -n -a 127.0.0.1 -p 8288 -f /data/mxlApp/bin/mxl_cgi
(gdb) set env LD_LIBRARY_PATH=/data/mxlApp/libs
(gdb) run #先不加载 coredump 文件,直接运行
Starting program: /usr/bin/spawn-fcgi -n -a 127.0.0.1 -p 8288 -f /data/mxlApp/bin/mxl_cgi
process 14767 is executing new program: /bin/bash
process 14767 is executing new program: /userdata/mxlApp/bin/mxl_cgi
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/libthread_db.so.1".
[New Thread 0xb6dc83e0 (LWP 14772)]
^C #按下按键 ctrl+C 暂停gdb 的运行
Thread 1 "mxl_cgi" received signal SIGINT, Interrupt.
0xb6fa17d4 in accept () from /lib/libpthread.so.0
(gdb) core-file core-mxl_cgi-3390-11-1692777034 #加载coredump 文件
A program is being debugged already. Kill it? (y or n) y
[New LWP 3391]
[New LWP 3390]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/libthread_db.so.1".
Core was generated by `/data/mxlApp/bin/mxl_cgi'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0xb6daaf90 in getenv () from /lib/libc.so.6
[Current thread is 1 (Thread 0xb6d123e0 (LWP 3391))]
(gdb) bt
#0 0xb6daaf90 in getenv () from /lib/libc.so.6
#1 0x00010d3c in pthread_run1 (arg=<optimized out>) at /home/mxl/tmp/wifimesh_app/gateway/mxl_cgi/web_server_app.c:39
#2 0xb6edfdf8 in start_thread () from /lib/libpthread.so.0
#3 0xb6e4eae8 in ?? () from /lib/libc.so.6
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb)