Debugging Android core dumps

Debugging Android core dumps

When a process crashes, a core dump (called Tombstone in Android) is printed out in the logcat and stored under /data/tombstones/tombstone_nn, where nn is just a counter.

To get some useful info out of it we need to dereference it with the source. The libraries on the target are stripped of symbols, so you’ll need the unstripped ones from /out/product/name/symbols/system/libwhatever.so.

Then you can use the addr2line located in ${android-src}/prebuilt/linux-x86/toolchain/arm-eabi-4.4.0/bin/arm-eabi-addr2line to dereference it.

Getting hands on, the crash I am looking at looks like:

*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
Build fingerprint: 'unknown'
pid: 1384, tid: 1384  >>> /system/bin/mediaserver <<<
signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr e08f3007
 r0 69708128  r1 00000000  r2 00000002  r3 00000000
 r4 00000000  r5 69708128  r6 69709214  r7 0000adf8
 r8 00000000  r9 00000000  10 00000000  fp 00000000
 ip 697108f8  sp 7e8fca08  lr e08f3003  pc 4013eb04  cpsr 80000010
 d0  968f6dec00000000  d1  6f20656c69709214
 d2  6e6f635f726f6c62  d3  2e6e6f697372652f
 d4  0000000000000000  d5  0000000000000000
 d6  3f80000000000000  d7  00000000ced2e124
 d8  0000000000000000  d9  0000000000000000
 d10 0000000000000000  d11 0000000000000000
 d12 0000000000000000  d13 0000000000000000
 d14 0000000000000000  d15 0000000000000000
 d16 0000000000000000  d17 0000000000000000
 d18 0000000000000000  d19 0000000000000000
 d20 0000000000000000  d21 0000000000000000
 d22 0000000000000000  d23 0000000000000000
 d24 0000000000000000  d25 0000000000000000
 d26 0000000000000000  d27 0000000000000000
 d28 0000000000000000  d29 0000000000000000
 d30 0000000000000000  d31 0000000000000000
 scr 00000010

         #00  pc 0003eb04  /system/lib/libasound.so
         #01  pc 0003edc8  /system/lib/libasound.so
         #02  pc 0000a9bc  /system/lib/libaudio.so
         #03  pc 0000c2d8  /system/lib/libaudio.so
         #04  pc 000234f8  /system/lib/libaudioflinger.so
         #05  pc 00029a06  /system/lib/libaudioflinger.so
         #06  pc 000088f2  /system/bin/mediaserver
         #07  pc 000089e6  /system/bin/mediaserver
         #08  pc 00014b52  /system/lib/libc.so

code around pc:
4013eae4 e1a0e00f e59cf000 e8bd8010 e92d4010
4013eaf4 e590e010 e1a03001 e3a02002 e3a01000
4013eb04 e59ec004 e1a0e00f e59cf000 e8bd8010
4013eb14 e590c010 e1a03002 e92d4010 e59c2008
4013eb24 e3120008 03e00015 08bd8010 e1a02001

code around lr:
e08f2fe0 ffffffff ffffffff ffffffff ffffffff
e08f2ff0 ffffffff ffffffff ffffffff ffffffff
e08f3000 ffffffff ffffffff ffffffff ffffffff
e08f3010 ffffffff ffffffff ffffffff ffffffff
e08f3020 ffffffff ffffffff ffffffff ffffffff

stack:
    7e8fc9c8  0000aed0
    7e8fc9cc  697107b8
    7e8fc9d0  0000afa0
    7e8fc9d4  6970c17f  /system/lib/libaudio.so
    7e8fc9d8  00000000
    7e8fc9dc  0000aa98
    7e8fc9e0  0000aaf8
    7e8fc9e4  0000aac8
    7e8fc9e8  68223204
    7e8fc9ec  0000b854
    7e8fc9f0  0000aaf8
    7e8fc9f4  00000000
    7e8fc9f8  0000001b
    7e8fc9fc  0000aeb8
    7e8fca00  df002777
    7e8fca04  e3a070ad
#00 7e8fca08  00000000
    7e8fca0c  4013edcc  /system/lib/libasound.so
#01 7e8fca10  0000adc8
    7e8fca14  3f800000
    7e8fca18  00000000
    7e8fca1c  6970a9c0  /system/lib/libaudio.so
#02 7e8fca20  0000adc8
    7e8fca24  6970c2db  /system/lib/libaudio.so
#03 7e8fca28  0000adc8
    7e8fca2c  68d234fb  /system/lib/libaudioflinger.so
#04 7e8fca30  0000adc8
    7e8fca34  00000000
    7e8fca38  00000000
    7e8fca3c  3f800000
    7e8fca40  00000000
    7e8fca44  68d29a0b  /system/lib/libaudioflinger.so
#05 7e8fca48  00000000
    7e8fca4c  00000000
    7e8fca50  00000000
    7e8fca54  00000000
    7e8fca58  00000000
    7e8fca5c  00000000
    7e8fca60  00000000
    7e8fca64  00000000
    7e8fca68  7e8fcab4
    7e8fca6c  00000000
    7e8fca70  00000000
    7e8fca74  00000000
    7e8fca78  00000000
    7e8fca7c  0000adc8
    7e8fca80  6821764d  /system/lib/libbinder.so
    7e8fca84  0000ad50
    7e8fca88  7e8fcaec
    7e8fca8c  000088f5  /system/bin/mediaserver
#06 7e8fca90  00000019
    7e8fca94  7e8fc660
    7e8fca98  0000ad98
    7e8fca9c  0000ad50
    7e8fcaa0  7e8fcab4
    7e8fcaa4  7e8fcae4
    7e8fcaa8  00000001
    7e8fcaac  000089eb  /system/bin/mediaserver
#07 7e8fcab0  00000001
    7e8fcab4  7e8fcae4
    7e8fcab8  0000ad50
    7e8fcabc  0000b848
    7e8fcac0  00008894  /system/bin/mediaserver
    7e8fcac4  6fd14b55  /system/lib/libc.so
#08 7e8fcac8  00000000
    7e8fcacc  00000000
    7e8fcad0  00000000
    7e8fcad4  00000000
    7e8fcad8  00000000
    7e8fcadc  ffffffe4
    7e8fcae0  00000001
    7e8fcae4  7e8fcbf0
    7e8fcae8  00000000
    7e8fcaec  7e8fcc08
    7e8fcaf0  7e8fcc45
    7e8fcaf4  7e8fcc6d
    7e8fcaf8  7e8fcc80
    7e8fcafc  7e8fcc95
    7e8fcb00  7e8fccb0
    7e8fcb04  7e8fccc3
    7e8fcb08  7e8fcce0
    7e8fcb0c  7e8fcd00

Which can be dereferenced as follows:

#00  pc 0003eb04  /system/lib/libasound.so

${android-src}/prebuilt/linux-x86/toolchain/arm-eabi-4.4.0/bin/arm-eabi-addr2line -f -e out/target/product/imx51_ccwmx51js/symbols/system/lib/libasound.so 0003eb04

snd_mixer_selem_has_playback_channel
/home/alex/Projects/android/external/alsa-lib/src/mixer/simple.c:270

I’ll omit the addr2line call from now on and just show the results.

#01  pc 0003edc8  /system/lib/libasound.so

snd_mixer_selem_set_playback_volume_all
/home/alex/Projects/android/external/alsa-lib/src/mixer/simple.c:478

#02  pc 0000a9bc  /system/lib/libaudio.so

_ZN7android9ALSAMixer15snd_mixer_selem_set_playback_volume_all
/home/alex/Projects/android/hardware/alsa_sound/ALSAMixer.cpp:237

#03  pc 0000c2d8  /system/lib/libaudio.so

_ZN7android18A2dpAudioInterface15setMasterVolumeEf
/home/alex/Projects/android/frameworks/base/services/audioflinger/A2dpAudioInterface.cpp
:210

#04  pc 000234f8  /system/lib/libaudioflinger.so

_ZN7android12AudioFlinger15setMasterVolumeEf
/home/alex/Projects/android/frameworks/base/services/audioflinger/AudioFlinger.cpp:445

#05  pc 00029a06  /system/lib/libaudioflinger.so

AudioFlinger
/home/alex/Projects/android/frameworks/base/services/audioflinger/AudioFlinger.cpp:144

#06  pc 000088f2  /system/bin/mediaserver

_ZN7android13BinderServiceINS_12AudioFlingerEE7publishEv
/home/alex/Projects/android/frameworks/base/include/binder/BinderService.h:39

#07  pc 000089e6  /system/bin/mediaserver

_ZN7android13BinderServiceINS_12AudioFlingerEE11instantiateEv
/home/alex/Projects/android/frameworks/base/include/binder/BinderService.h:50

#08  pc 00014b52  /system/lib/libc.so

__libc_init
/home/alex/Projects/android/bionic/libc/bionic/libc_init_dynamic.c:114

Update: If you want a semiautomatic way, try the following script:

#!/bin/bash

# $1 is the name of the file including the dump
# $2 is the name of the library
# parse_stack.py needs to be in your path

# Customize
ANDROID_BASE=
PRODUCT=

# Do not change after this line

SOLIB_NAME=$2
PATH_TO_SOLIB=${ANDROID_BASE}/out/target/product/${PRODUCT}/symbols/system/lib/${SOLIB_NAME}

${ANDROID_BASE}/prebuilt/linux-x86/toolchain/arm-eabi-4.4.0/bin/arm-eabi-objdump -S ${PATH_TO_SOLIB} > ${SOLIB_NAME}.arm
parse_stack.py ${SOLIB_NAME}.arm $1
rm ${SOLIB_NAME}.arm

With parse_stack.py in your path:

#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import re
import os

sohead = re.compile('(.+\.so):')
funchead = re.compile('([0-9a-f]{8}) <(.+)>:')
funcline = re.compile('^[ ]+([0-9a-f]+):.+')

def parsestack( lines, libname ):
    crashline = re.compile('.+pc.([0-9a-f]{8}).+%s' % libname )
    ret = []
    for l in lines:
        m = crashline.match(l)
        if m:
            addr =  m.groups()[0]
            ret.append(int(addr,16))
    return ret

def parseasm( lines ):
    ret = []
    current = None
    restartcode = False;
    for l in lines:
        m = funchead.match(l)
        if m:
            if current:
                ret.append(current)
            startaddr, funcname =  m.groups()
            current = [ funcname, int(startaddr,16), int(startaddr,16), int(startaddr,16), [] ]
            continue
        m = funcline.match(l)
        if m:
            restartcode = True;
            addr =  m.groups()[0]
            if current != None:
                current[3] = int(addr,16)
            continue
        m = sohead.match(l)
        if m:
            so =  m.groups()[0]
            so = os.path.split(so)[1]
            continue

        #Assume anything else is a code line
        if restartcode:
#            print 'XXX',l
            restartcode = False;
            ret.append(current)
            current = [ current[0], current[1], current[3], current[3], [] ]
        if current != None:
            current[4].append(l);

    return so, ret

if __name__=="__main__":
    asm, stack = sys.argv[1],sys.argv[2]

    libname, asm = parseasm( file(asm).read().split('\n') )
    stack = parsestack( file(stack).read().split('\n'), libname )
    for addr in stack:
        for func, funcstart, segstart, segend, code in asm:
            if addr >= segstart and addr <= segend:
                print "0x%08x:%32s + 0x%04x %s" % ( addr, func, addr-funcstart, "".join(["\n"+x for x in code]))

To use it you will need a file containing a dump. You can get it from logcat with:

user@computer:$ adb logcat -d > logcat.txt

Using the one above as reference, we would call the script with:

user@computer:$ android_stack_dump.sh my.dump libasound.so
android_stack_dump.sh my.dump libaudio.so
android_stack_dump.sh my.dump libaudioflinger.so
android_stack_dump.sh my.dump libc.so