Spark streaming应用运行7天之后，自动退出，日志显示token for xxx(用户名): HDFS_DELEGATION_TOKEN owner=xxxx@xxxx.com, renewer=yarn, realUser=, issueDate=1581323654722, maxDate=1581928454722, sequenceNumber=6445344, masterKeyId=1583) is expired, current time: 2020-02-17 16:37:40,567+0800 expected renewal time: 2020-02-17 16:34:14,722+0800 ，可是提交应用的已经用kinit获取了kerberos票据，从日志信息中可以看出，是spark streaming的checkpoint操作hadoop时，发现kerberos票据过期导致。
解决方案：
spark-submit 其他参数。。。 --keytab /home/keytabs/xxxx.keytab --principal xxxx --conf spark.hadoop.fs.hdfs.impl.disable.cache=true
xxxx只是为了隐藏真实用户名 还有就是程序提交时肯定还有其他参数需要配置的。
完整异常信息如下：
暂时隐去用户名信息
20/02/17 16:37:40 ERROR util.Utils: Uncaught exception in thread Thread-5
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager
I
n
v
a
l
i
d
T
o
k
e
n
)
:
t
o
k
e
n
(
t
o
k
e
n
f
o
r
h
a
d
o
o
p
:
H
D
F
S
D
E
L
E
G
A
T
I
O
N
T
O
K
E
N
o
w
n
e
r
=
x
x
x
x
@
x
x
x
x
.
c
o
m
,
r
e
n
e
w
e
r
=
y
a
r
n
,
r
e
a
l
U
s
e
r
=
,
i
s
s
u
e
D
a
t
e
=
1581323654722
,
m
a
x
D
a
t
e
=
1581928454722
,
s
e
q
u
e
n
c
e
N
u
m
b
e
r
=
6445344
,
m
a
s
t
e
r
K
e
y
I
d
=
1583
)
i
s
e
x
p
i
r
e
d
,
c
u
r
r
e
n
t
t
i
m
e
:
2020
−
02
−
1716
:
37
:
40
,
567
+
0800
e
x
p
e
c
t
e
d
r
e
n
e
w
a
l
t
i
m
e
:
2020
−
02
−
1716
:
34
:
14
,
722
+
0800
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
i
p
c
.
C
l
i
e
n
t
.
c
a
l
l
(
C
l
i
e
n
t
.
j
a
v
a
:
1504
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
i
p
c
.
C
l
i
e
n
t
.
c
a
l
l
(
C
l
i
e
n
t
.
j
a
v
a
:
1441
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
i
p
c
.
P
r
o
t
o
b
u
f
R
p
c
E
n
g
i
n
e
InvalidToken): token (token for hadoop: HDFS_DELEGATION_TOKEN owner=xxxx@xxxx.com, renewer=yarn, realUser=, issueDate=1581323654722, maxDate=1581928454722, sequenceNumber=6445344, masterKeyId=1583) is expired, current time: 2020-02-17 16:37:40,567+0800 expected renewal time: 2020-02-17 16:34:14,722+0800 at org.apache.hadoop.ipc.Client.call(Client.java:1504) at org.apache.hadoop.ipc.Client.call(Client.java:1441) at org.apache.hadoop.ipc.ProtobufRpcEngine
InvalidToken):token(tokenforhadoop:HDFSDELEGATIONTOKENowner=xxxx@xxxx.com,renewer=yarn,realUser=,issueDate=1581323654722,maxDate=1581928454722,sequenceNumber=6445344,masterKeyId=1583)isexpired,currenttime:2020−02−1716:37:40,567+0800expectedrenewaltime:2020−02−1716:34:14,722+0800atorg.apache.hadoop.ipc.Client.call(Client.java:1504)atorg.apache.hadoop.ipc.Client.call(Client.java:1441)atorg.apache.hadoop.ipc.ProtobufRpcEngineInvoker.invoke(ProtobufRpcEngine.java:230)
at com.sun.proxy.
P
r
o
x
y
16.
g
e
t
F
i
l
e
I
n
f
o
(
U
n
k
n
o
w
n
S
o
u
r
c
e
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
p
r
o
t
o
c
o
l
P
B
.
C
l
i
e
n
t
N
a
m
e
n
o
d
e
P
r
o
t
o
c
o
l
T
r
a
n
s
l
a
t
o
r
P
B
.
g
e
t
F
i
l
e
I
n
f
o
(
C
l
i
e
n
t
N
a
m
e
n
o
d
e
P
r
o
t
o
c
o
l
T
r
a
n
s
l
a
t
o
r
P
B
.
j
a
v
a
:
771
)
a
t
s
u
n
.
r
e
f
l
e
c
t
.
G
e
n
e
r
a
t
e
d
M
e
t
h
o
d
A
c
c
e
s
s
o
r
34.
i
n
v
o
k
e
(
U
n
k
n
o
w
n
S
o
u
r
c
e
)
a
t
s
u
n
.
r
e
f
l
e
c
t
.
D
e
l
e
g
a
t
i
n
g
M
e
t
h
o
d
A
c
c
e
s
s
o
r
I
m
p
l
.
i
n
v
o
k
e
(
D
e
l
e
g
a
t
i
n
g
M
e
t
h
o
d
A
c
c
e
s
s
o
r
I
m
p
l
.
j
a
v
a
:
43
)
a
t
j
a
v
a
.
l
a
n
g
.
r
e
f
l
e
c
t
.
M
e
t
h
o
d
.
i
n
v
o
k
e
(
M
e
t
h
o
d
.
j
a
v
a
:
606
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
i
o
.
r
e
t
r
y
.
R
e
t
r
y
I
n
v
o
c
a
t
i
o
n
H
a
n
d
l
e
r
.
i
n
v
o
k
e
M
e
t
h
o
d
(
R
e
t
r
y
I
n
v
o
c
a
t
i
o
n
H
a
n
d
l
e
r
.
j
a
v
a
:
258
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
i
o
.
r
e
t
r
y
.
R
e
t
r
y
I
n
v
o
c
a
t
i
o
n
H
a
n
d
l
e
r
.
i
n
v
o
k
e
(
R
e
t
r
y
I
n
v
o
c
a
t
i
o
n
H
a
n
d
l
e
r
.
j
a
v
a
:
104
)
a
t
c
o
m
.
s
u
n
.
p
r
o
x
y
.
Proxy16.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771) at sun.reflect.GeneratedMethodAccessor34.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:258) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104) at com.sun.proxy.
Proxy16.getFileInfo(UnknownSource)atorg.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771)atsun.reflect.GeneratedMethodAccessor34.invoke(UnknownSource)atsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)atjava.lang.reflect.Method.invoke(Method.java:606)atorg.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:258)atorg.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:104)atcom.sun.proxy.Proxy17.getFileInfo(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2126)
at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1262)
at org.apache.hadoop.hdfs.DistributedFileSystem
20.
d
o
C
a
l
l
(
D
i
s
t
r
i
b
u
t
e
d
F
i
l
e
S
y
s
t
e
m
.
j
a
v
a
:
1258
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
f
s
.
F
i
l
e
S
y
s
t
e
m
L
i
n
k
R
e
s
o
l
v
e
r
.
r
e
s
o
l
v
e
(
F
i
l
e
S
y
s
t
e
m
L
i
n
k
R
e
s
o
l
v
e
r
.
j
a
v
a
:
81
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
d
f
s
.
D
i
s
t
r
i
b
u
t
e
d
F
i
l
e
S
y
s
t
e
m
.
g
e
t
F
i
l
e
S
t
a
t
u
s
(
D
i
s
t
r
i
b
u
t
e
d
F
i
l
e
S
y
s
t
e
m
.
j
a
v
a
:
1258
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
f
s
.
F
i
l
e
S
y
s
t
e
m
.
e
x
i
s
t
s
(
F
i
l
e
S
y
s
t
e
m
.
j
a
v
a
:
1418
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
E
v
e
n
t
L
o
g
g
i
n
g
L
i
s
t
e
n
e
r
.
s
t
o
p
(
E
v
e
n
t
L
o
g
g
i
n
g
L
i
s
t
e
n
e
r
.
s
c
a
l
a
:
232
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
S
p
a
r
k
C
o
n
t
e
x
t
20.doCall(DistributedFileSystem.java:1258) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1258) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1418) at org.apache.spark.scheduler.EventLoggingListener.stop(EventLoggingListener.scala:232) at org.apache.spark.SparkContext
20.doCall(DistributedFileSystem.java:1258)atorg.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)atorg.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1258)atorg.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1418)atorg.apache.spark.scheduler.EventLoggingListener.stop(EventLoggingListener.scala:232)atorg.apache.spark.SparkContext
a
n
o
n
f
u
n
anonfun
anonfunstop
7
7
7
a
n
o
n
f
u
n
anonfun
anonfunapply
m
c
V
mcV
mcVsp
5.
a
p
p
l
y
(
S
p
a
r
k
C
o
n
t
e
x
t
.
s
c
a
l
a
:
1831
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
S
p
a
r
k
C
o
n
t
e
x
t
5.apply(SparkContext.scala:1831) at org.apache.spark.SparkContext
5.apply(SparkContext.scala:1831)atorg.apache.spark.SparkContext
a
n
o
n
f
u
n
anonfun
anonfunstop
7
7
7
a
n
o
n
f
u
n
anonfun
anonfunapply
m
c
V
mcV
mcVsp
5.
a
p
p
l
y
(
S
p
a
r
k
C
o
n
t
e
x
t
.
s
c
a
l
a
:
1831
)
a
t
s
c
a
l
a
.
O
p
t
i
o
n
.
f
o
r
e
a
c
h
(
O
p
t
i
o
n
.
s
c
a
l
a
:
257
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
S
p
a
r
k
C
o
n
t
e
x
t
5.apply(SparkContext.scala:1831) at scala.Option.foreach(Option.scala:257) at org.apache.spark.SparkContext
5.apply(SparkContext.scala:1831)atscala.Option.foreach(Option.scala:257)atorg.apache.spark.SparkContext
a
n
o
n
f
u
n
anonfun
anonfunstop
7.
a
p
p
l
y
7.apply
7.applymcV
s
p
(
S
p
a
r
k
C
o
n
t
e
x
t
.
s
c
a
l
a
:
1831
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
u
t
i
l
.
U
t
i
l
s
sp(SparkContext.scala:1831) at org.apache.spark.util.Utils
sp(SparkContext.scala:1831)atorg.apache.spark.util.Utils.tryLogNonFatalError(Utils.scala:1295)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1830)
at org.apache.spark.SparkContextKaTeX parse error: Can't use function '$' in math mode at position 8: anonfun$̲2.apply$mcV$sp(…anonfun$runAll
1
1
1
a
n
o
n
f
u
n
anonfun
anonfunapply
m
c
V
mcV
mcVsp
1.
a
p
p
l
y
1.apply
1.applymcV
s
p
(
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
.
s
c
a
l
a
:
188
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
u
t
i
l
.
S
p
a
r
k
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
sp(ShutdownHookManager.scala:188) at org.apache.spark.util.SparkShutdownHookManager
sp(ShutdownHookManager.scala:188)atorg.apache.spark.util.SparkShutdownHookManager
a
n
o
n
f
u
n
anonfun
anonfunrunAll
1
1
1
a
n
o
n
f
u
n
anonfun
anonfunapply
m
c
V
mcV
mcVsp
1.
a
p
p
l
y
(
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
.
s
c
a
l
a
:
188
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
u
t
i
l
.
S
p
a
r
k
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
1.apply(ShutdownHookManager.scala:188) at org.apache.spark.util.SparkShutdownHookManager
1.apply(ShutdownHookManager.scala:188)atorg.apache.spark.util.SparkShutdownHookManager
a
n
o
n
f
u
n
anonfun
anonfunrunAll
1
1
1
a
n
o
n
f
u
n
anonfun
anonfunapply
m
c
V
mcV
mcVsp
1.
a
p
p
l
y
(
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
.
s
c
a
l
a
:
188
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
u
t
i
l
.
U
t
i
l
s
1.apply(ShutdownHookManager.scala:188) at org.apache.spark.util.Utils
1.apply(ShutdownHookManager.scala:188)atorg.apache.spark.util.Utils.logUncaughtExceptions(Utils.scala:1963)
at org.apache.spark.util.SparkShutdownHookManagerKaTeX parse error: Can't use function '$' in math mode at position 8: anonfun$̲runAll$1.apply$…anonfun$runAll
1.
a
p
p
l
y
(
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
.
s
c
a
l
a
:
188
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
u
t
i
l
.
S
p
a
r
k
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
1.apply(ShutdownHookManager.scala:188) at org.apache.spark.util.SparkShutdownHookManager
1.apply(ShutdownHookManager.scala:188)atorg.apache.spark.util.SparkShutdownHookManager
a
n
o
n
f
u
n
anonfun
anonfunrunAll
1.
a
p
p
l
y
(
S
h
u
t
d
o
w
n
H
o
o
k
M
a
n
a
g
e
r
.
s
c
a
l
a
:
188
)
a
t
s
c
a
l
a
.
u
t
i
l
.
T
r
y
1.apply(ShutdownHookManager.scala:188) at scala.util.Try
1.apply(ShutdownHookManager.scala:188)atscala.util.Try.apply(Try.scala:192)
at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188)
at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178)
at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
4392
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
