我们经常碰到一些定时任务卡死或者执行时间很长,这样的问题我们排查手段比较常用的是jstack命令
来查看线程堆栈,然后根据我们监控中的threadId或者threadName来查找线程详细堆栈看卡在哪个方法。
常用命令如下:
jstack pid >jstack.log
现在我们可以提供界面查看卡死线程堆栈
controller代码
@RequestMapping("/getStackTrace")
@ResponseBody
public String getStackStace(HttpServletRequest request, HttpServletResponse response) {
logger.debug("执行选择的Job对象 queryQuartzLog quartzLog:");
try {
String threadId=request.getParameter("threadId");
if(StringUtils.isBlank(threadId)){
return "";
}
ThreadInfo info = ThreadUtilities.getExtendThreadInfo(Long.valueOf(threadId));
List<String> list=new ArrayList<String>();
StackTraceElement[] st=info.getStackTrace();
int index=0;
MonitorInfo[] monitors=info.getLockedMonitors();
for(StackTraceElement e:st){
StringBuffer sb=new StringBuffer();
if(index>0){
sb.append(" <- ");
sb.append(System.getProperty("line.separator"));
}
sb.append(e.toString()+ "\n");
list.add(sb.toString());
if (monitors != null) {
for (MonitorInfo mi : monitors) {
if (mi.getLockedStackDepth() == index) {
list.add(" <- - locked "+mi.toString());
}
}
}
index++;
}
return generateHtml("vm/getStackTrace.vm", list);
} catch (Exception e) {
logger.error("doRunQuartzJob 执行选择的Job对象出错", e);
return "fail";
}
}
ThreadUtilities.getExtendThreadInfo
public static ThreadInfo getExtendThreadInfo(final long threadID) {
ThreadInfo ti = null;
final ThreadMXBean threadMBean = ManagementFactory.getThreadMXBean();
if (threadMBean.isObjectMonitorUsageSupported()) {
// VMs that support the monitor usage monitoring
ThreadInfo[] infos = threadMBean.dumpAllThreads(true, false);
for (ThreadInfo info : infos) {
if (info.getThreadId() == threadID) {
ti = info;
break;
}
}
} else {
// VM doesn't support monitor usage monitoring
ti = threadMBean.getThreadInfo(threadID, Integer.MAX_VALUE);
}
return ti;
}
然后通过vm代码将堆栈展示在页面上
getStackTrace.vm
<!DOCTYPE html>
<html lang="zh-cn">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
<meta name="description" content="" />
<meta name="Keywords" content="" />
<title>job logs</title>
</head>
<body>
<div>
#foreach($element in $data)
$element<br/>
#end
</div>
</body>
</html>
public class BaseController {
private static final VelocityEngine ve = new VelocityEngine();
private static final Logger logger = LoggerFactory.getLogger(QuartzController.class);
static{
// 初始化Velocity引擎
InputStream is = BaseController.class.getResourceAsStream("/velocity.properties");
Properties properties = new Properties();
try {
properties.load(is);
} catch (IOException e) {
e.printStackTrace();
}
ve.init(properties);
}
public String generateHtml(String vmTemplate,Object obj){
StringWriter sw = new StringWriter();
try {
VelocityContext ctx = new VelocityContext();
ctx.put("data", obj);
ve.getTemplate(vmTemplate,"UTF-8").merge(ctx, sw);
} catch (Exception e) {
logger.error("转化vm模板出错",e);
}finally{
if(sw!=null){
try {
sw.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return sw.toString();
}
}