JVM崩溃,原因是触发了JVM代码的bug。一般的解决办法是升级到最新的小版本上。不过有些特殊的场景,是探针agent导致,需要分析hs_err_pid.log中的堆栈信息。
server.log:
<2020-4-17 下午07时25分21秒 CST> <Started WebLogic Managed Server “GG_APP02_6001” for domain “GG_domain” running in Production Mode>
jowto : weblogic/servlet/internal/ServletResponseImpl
System property [org.owasp.esapi.opsteam] is not set
System property [org.owasp.esapi.devteam] is not set
Attempting to load ESAPI.properties via file I/O.
Loaded ‘ESAPI.properties’ properties file
SecurityConfiguration for Validator.ConfigurationFile.MultiValued not found in ESAPI.properties. Using default: false
Attempting to load validation.properties via file I/O.
Loaded ‘validation.properties’ properties file
#
# A fatal error has been detected by the Java Runtime Environment:
#
# SIGSEGV (0xb) at pc=0x00007fb5c2393491, pid=20806, tid=140418263463680
#
# JRE version: 6.0_39-b04
# Java VM: Java HotSpot(TM) 64-Bit Server VM (20.14-b01 mixed mode linux-amd64 compressed oops)
# Problematic frame:
# C [libJSPAgent.so+0xd8491] CBasicProtectionCheck::pre_check2(_Jv_JNIEnv*, __jclass*, __jobject*, request_info_t&)+0x261
#
# An error report file with more information is saved as:
# /app/CCCdomains/GG_domain/hs_err_pid20806.log
#
# If you would like to submit a bug report, please visit:
# http://java.sun.com/webapps/bugreport/crash.jsp
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.
#
/app/CCCdomains/GG_domain/bin/startWebLogic.sh: line 193: 20806 已放弃 (core dumped) ${JAVA_HOME}/bin/java ${JAVA_VM} M E M A R G S − D w e b l o g i c . N a m e = {MEM_ARGS} -Dweblogic.Name= MEMARGS−Dweblogic.Name={SERVER_NAME} -Djava.security.policy=${WL_HOME}/server/lib/weblogic.policy ${JAVA_OPTIONS} ${PROXY_SETTINGS} ${SERVER_CLASS}
/app/CCCdomains/GG_domain/hs_err_pid20806.log:
Register to memory mapping:
RAX=0x00000032be4f32d8: _ZNSs4_Rep20_S_empty_rep_storageE+0x18 in /usr/lib64/libstdc++.so.6 at 0x00000032be200000
RBX=0x00000032be4f32c0: _ZNSs4_Rep20_S_empty_rep_storageE+0 in /usr/lib64/libstdc++.so.6 at 0x00000032be200000
RCX=0x00007fb5acb69640 is pointing into the stack for thread: 0x00007fb626b48800
RDX=0x00000032be4f32d8: _ZNSs4_Rep20_S_empty_rep_storageE+0x18 in /usr/lib64/libstdc++.so.6 at 0x00000032be200000
RSP=0x00007fb5acb695f0 is pointing into the stack for thread: 0x00007fb626b48800
RBP=0x00000032be4f32c0: _ZNSs4_Rep20_S_empty_rep_storageE+0 in /usr/lib64/libstdc++.so.6 at 0x00000032be200000
RSI=0x00007fb5acb69720 is pointing into the stack for thread: 0x00007fb626b48800
RDI=0x0000000000000000 is an unknown value
R8 =0x0000000000000ffc is an unknown value
R9 =0x00007fb62c3653d8: <offset 0xa9e3d8> in /app/CCC_jdk1.6.0_39/jre/lib/amd64/server/libjvm.so at 0x00007fb62b8c7000
R10=0x00007fb62c369278: <offset 0xaa2278> in /app/CCC_jdk1.6.0_39/jre/lib/amd64/server/libjvm.so at 0x00007fb62b8c7000
R11=0x00007fb62c3653e8: <offset 0xa9e3e8> in /app/CCC_jdk1.6.0_39/jre/lib/amd64/server/libjvm.so at 0x00007fb62b8c7000
R12=0x00007fb5acb69720 is pointing into the stack for thread: 0x00007fb626b48800
R13=0x00007fb5c2718640: <offset 0x45d640> in /app/CCC_domains/gg_domain/jspAgent/x64/libJSPAgent.so at 0x00007fb5c22bb000
R14=0x00007fb5acb69768 is pointing into the stack for thread: 0x00007fb626b48800
R15=0x00007fb5acb69938 is pointing into the stack for thread: 0x00007fb626b48800
Stack: [0x00007fb5aca6b000,0x00007fb5acb6c000], sp=0x00007fb5acb695f0, free space=1017k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
C [libJSPAgent.so+0xd8491] CBasicProtectionCheck::pre_check2(_Jv_JNIEnv*, __jclass*, __jobject*, request_info_t&)+0x261
Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
j com.jowto.agent.AgentBridge.specialCheckSql(Ljava/lang/Object;Ljava/lang/Class;Ljava/lang/String;)Z+0
j com.jowto.agent.AgentBridge.getReadData([BI)V+82
j weblogic.servlet.internal.ServletInputStreamImpl.read([BII)I+35
J java.io.FilterInputStream.read([BII)I
J java.io.ObjectInputStream
P
e
e
k
I
n
p
u
t
S
t
r
e
a
m
.
r
e
a
d
F
u
l
l
y
(
[
B
I
I
)
V
j
j
a
v
a
.
i
o
.
O
b
j
e
c
t
I
n
p
u
t
S
t
r
e
a
m
PeekInputStream.readFully([BII)V j java.io.ObjectInputStream
PeekInputStream.readFully([BII)Vjjava.io.ObjectInputStreamBlockDataInputStream.readLong()J+23
j java.io.ObjectInputStream.readLong()J+4
j java.io.ObjectStreamClass.readNonProxy(Ljava/io/ObjectInputStream;)V+10
j java.io.ObjectInputStream.readClassDescriptor()Ljava/io/ObjectStreamClass;+10
J java.io.ObjectInputStream.readObject0(Z)Ljava/lang/Object;
J java.io.ObjectInputStream.readObject()Ljava/lang/Object;
j weblogic.deploy.service.internal.transport.http.DeploymentServiceServlet.handleDeploymentServiceMessage(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;Lweblogic/security/acl/internal/AuthenticatedSubject;)V+203
j weblogic.deploy.service.internal.transport.http.DeploymentServiceServlet.internalDoPost(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;Lweblogic/security/acl/internal/AuthenticatedSubject;)V+27
j weblogic.deploy.service.internal.transport.http.DeploymentServiceServlet.access$000(Lweblogic/deploy/service/internal/transport/http/DeploymentServiceServlet;Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;Lweblogic/security/acl/internal/AuthenticatedSubject;)V+4
j weblogic.deploy.service.internal.transport.http.DeploymentServiceServlet
1.
r
u
n
(
)
L
j
a
v
a
/
l
a
n
g
/
O
b
j
e
c
t
;
+
16
j
w
e
b
l
o
g
i
c
.
s
e
c
u
r
i
t
y
.
a
c
l
.
i
n
t
e
r
n
a
l
.
A
u
t
h
e
n
t
i
c
a
t
e
d
S
u
b
j
e
c
t
.
d
o
A
s
(
L
w
e
b
l
o
g
i
c
/
s
e
c
u
r
i
t
y
/
s
u
b
j
e
c
t
/
A
b
s
t
r
a
c
t
S
u
b
j
e
c
t
;
L
j
a
v
a
/
s
e
c
u
r
i
t
y
/
P
r
i
v
i
l
e
g
e
d
E
x
c
e
p
t
i
o
n
A
c
t
i
o
n
;
)
L
j
a
v
a
/
l
a
n
g
/
O
b
j
e
c
t
;
+
34
j
w
e
b
l
o
g
i
c
.
s
e
c
u
r
i
t
y
.
s
e
r
v
i
c
e
.
S
e
c
u
r
i
t
y
M
a
n
a
g
e
r
.
r
u
n
A
s
(
L
w
e
b
l
o
g
i
c
/
s
e
c
u
r
i
t
y
/
a
c
l
/
i
n
t
e
r
n
a
l
/
A
u
t
h
e
n
t
i
c
a
t
e
d
S
u
b
j
e
c
t
;
L
w
e
b
l
o
g
i
c
/
s
e
c
u
r
i
t
y
/
a
c
l
/
i
n
t
e
r
n
a
l
/
A
u
t
h
e
n
t
i
c
a
t
e
d
S
u
b
j
e
c
t
;
L
j
a
v
a
/
s
e
c
u
r
i
t
y
/
P
r
i
v
i
l
e
g
e
d
E
x
c
e
p
t
i
o
n
A
c
t
i
o
n
;
)
L
j
a
v
a
/
l
a
n
g
/
O
b
j
e
c
t
;
+
18
j
w
e
b
l
o
g
i
c
.
d
e
p
l
o
y
.
s
e
r
v
i
c
e
.
i
n
t
e
r
n
a
l
.
t
r
a
n
s
p
o
r
t
.
h
t
t
p
.
D
e
p
l
o
y
m
e
n
t
S
e
r
v
i
c
e
S
e
r
v
l
e
t
.
d
o
P
o
s
t
(
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
h
t
t
p
/
H
t
t
p
S
e
r
v
l
e
t
R
e
q
u
e
s
t
;
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
h
t
t
p
/
H
t
t
p
S
e
r
v
l
e
t
R
e
s
p
o
n
s
e
;
)
V
+
71
j
j
a
v
a
x
.
s
e
r
v
l
e
t
.
h
t
t
p
.
H
t
t
p
S
e
r
v
l
e
t
.
s
e
r
v
i
c
e
(
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
h
t
t
p
/
H
t
t
p
S
e
r
v
l
e
t
R
e
q
u
e
s
t
;
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
h
t
t
p
/
H
t
t
p
S
e
r
v
l
e
t
R
e
s
p
o
n
s
e
;
)
V
+
139
j
j
a
v
a
x
.
s
e
r
v
l
e
t
.
h
t
t
p
.
H
t
t
p
S
e
r
v
l
e
t
.
s
e
r
v
i
c
e
(
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
S
e
r
v
l
e
t
R
e
q
u
e
s
t
;
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
S
e
r
v
l
e
t
R
e
s
p
o
n
s
e
;
)
V
+
40
j
w
e
b
l
o
g
i
c
.
s
e
r
v
l
e
t
.
i
n
t
e
r
n
a
l
.
S
t
u
b
S
e
c
u
r
i
t
y
H
e
l
p
e
r
1.run()Ljava/lang/Object;+16 j weblogic.security.acl.internal.AuthenticatedSubject.doAs(Lweblogic/security/subject/AbstractSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;+34 j weblogic.security.service.SecurityManager.runAs(Lweblogic/security/acl/internal/AuthenticatedSubject;Lweblogic/security/acl/internal/AuthenticatedSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;+18 j weblogic.deploy.service.internal.transport.http.DeploymentServiceServlet.doPost(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V+71 j javax.servlet.http.HttpServlet.service(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V+139 j javax.servlet.http.HttpServlet.service(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+40 j weblogic.servlet.internal.StubSecurityHelper
1.run()Ljava/lang/Object;+16jweblogic.security.acl.internal.AuthenticatedSubject.doAs(Lweblogic/security/subject/AbstractSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;+34jweblogic.security.service.SecurityManager.runAs(Lweblogic/security/acl/internal/AuthenticatedSubject;Lweblogic/security/acl/internal/AuthenticatedSubject;Ljava/security/PrivilegedExceptionAction;)Ljava/lang/Object;+18jweblogic.deploy.service.internal.transport.http.DeploymentServiceServlet.doPost(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V+71jjavax.servlet.http.HttpServlet.service(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V+139jjavax.servlet.http.HttpServlet.service(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+40jweblogic.servlet.internal.StubSecurityHelperServletServiceAction.run()Ljava/lang/Object;+43
j weblogic.servlet.internal.StubSecurityHelper.invokeServlet(Ljavax/servlet/ServletRequest;Ljavax/servlet/http/HttpServletRequest;Lweblogic/servlet/internal/ServletRequestImpl;Ljavax/servlet/ServletResponse;Ljavax/servlet/http/HttpServletResponse;Ljavax/servlet/Servlet;)Ljava/lang/Throwable;+110
j weblogic.servlet.internal.ServletStubImpl.execute(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;Lweblogic/servlet/internal/FilterChainImpl;)V+467
j weblogic.servlet.internal.ServletStubImpl.execute(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+4
j weblogic.servlet.internal.WebAppServletContext
S
e
r
v
l
e
t
I
n
v
o
c
a
t
i
o
n
A
c
t
i
o
n
.
w
r
a
p
R
u
n
(
L
w
e
b
l
o
g
i
c
/
s
e
r
v
l
e
t
/
i
n
t
e
r
n
a
l
/
S
e
r
v
l
e
t
S
t
u
b
I
m
p
l
;
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
h
t
t
p
/
H
t
t
p
S
e
r
v
l
e
t
R
e
q
u
e
s
t
;
L
j
a
v
a
x
/
s
e
r
v
l
e
t
/
h
t
t
p
/
H
t
t
p
S
e
r
v
l
e
t
R
e
s
p
o
n
s
e
;
)
L
j
a
v
a
/
l
a
n
g
/
O
b
j
e
c
t
;
+
269
j
w
e
b
l
o
g
i
c
.
s
e
r
v
l
e
t
.
i
n
t
e
r
n
a
l
.
W
e
b
A
p
p
S
e
r
v
l
e
t
C
o
n
t
e
x
t
ServletInvocationAction.wrapRun(Lweblogic/servlet/internal/ServletStubImpl;Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)Ljava/lang/Object;+269 j weblogic.servlet.internal.WebAppServletContext
ServletInvocationAction.wrapRun(Lweblogic/servlet/internal/ServletStubImpl;Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)Ljava/lang/Object;+269jweblogic.servlet.internal.WebAppServletContextServletInvocationAction.run()Ljava/lang/Object;+13
j weblogic.security.acl.internal.AuthenticatedSubject.doAs(Lweblogic/security/subject/AbstractSubject;Ljava/security/PrivilegedAction;)Ljava/lang/Object;+34
j weblogic.security.service.SecurityManager.runAs(Lweblogic/security/acl/internal/AuthenticatedSubject;Lweblogic/security/acl/internal/AuthenticatedSubject;Ljava/security/PrivilegedAction;)Ljava/lang/Object;+18
j weblogic.servlet.internal.WebAppServletContext.securedExecute(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;Z)V+290
j weblogic.servlet.internal.WebAppServletContext.execute(Lweblogic/servlet/internal/ServletRequestImpl;Lweblogic/servlet/internal/ServletResponseImpl;)V+279
j weblogic.servlet.internal.ServletRequestImpl.run()V+295
j weblogic.work.ExecuteThread.execute(Ljava/lang/Runnable;)V+34
j weblogic.work.ExecuteThread.run()V+42
v ~StubRoutines::call_stub