1
INSERT OVERWRITE TABLE SJZL.DWD_PRD_IDRY_MONTH PARTITION
(PROV_ID, MONTH_ID)
SELECT LATN_ID,
PRVNCE_ID AS STD_PRVNCE_CD,
LATN_ID AS STD_LATN_CD,
PROD_INST_ID,
IDRY_APP_ID,
IDRY_APP_NM,
IDRY_APP_CATE_CD,
EFF_DATE,
EXP_DATE,
IDRY_STATUS,
RETRANS_NBR,
SEIAL_NBR,
BATCH_NBR,
from_unixtime(unix_timestamp(), ''yyyy/MM/dd HH:mm:ss''),
PRVNCE_ID,
MONTH_ID
FROM SJZL.DWS_PRD_IDRY_MONTH A1
WHERE MONTH_ID = ''201902''
AND PRVNCE_ID = ''811''
', 'application_1546863598863_14357797', '1', '811')
mysql connect success !
insert log successful
RuntimeError: Error: Traceback (most recent call last):
File “p_dwd_prd_idry_month_01.py”, line 111, in p_dwd_prd_idry_month
sqlContext.sql(sql_01)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/pyspark.zip/pyspark/sql/context.py”, line 580, in sql
return DataFrame(self._ssql_ctx.sql(sqlQuery), self)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/py4j-0.9-src.zip/py4j/java_gateway.py”, line 813, in call
answer, self.gateway_client, self.target_id, self.name)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/pyspark.zip/pyspark/sql/utils.py”, line 45, in deco
return f(*a, **kw)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/py4j-0.9-src.zip/py4j/protocol.py”, line 308, in get_return_value
format(target_id, “.”, name), value)
Py4JJavaError: An error occurred while calling o41.sql.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 3, A5-402-NF5280M4-2017-562.BIGDATA.CHINATELECOM.CN): org.apache.hadoop.fs.viewfs.NotInMountpointException: getDefaultReplication on empty path is invalid
at org.apache.hadoop.fs.viewfs.ViewFileSystem.getDefaultReplication(ViewFileSystem.java:593)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.getStream(WriterImpl.java:2103)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.flushStripe(WriterImpl.java:2120)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.checkMemory(WriterImpl.java:352)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.notifyWriters(MemoryManager.java:168)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.addedRow(MemoryManager.java:157)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.addRow(WriterImpl.java:2413)
at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
O
r
c
R
e
c
o
r
d
W
r
i
t
e
r
.
w
r
i
t
e
(
O
r
c
O
u
t
p
u
t
F
o
r
m
a
t
.
j
a
v
a
:
86
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
h
i
v
e
.
e
x
e
c
u
t
i
o
n
.
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
OrcRecordWriter.write(OrcOutputFormat.java:86) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable
OrcRecordWriter.write(OrcOutputFormat.java:86)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable
a
n
o
n
f
u
n
anonfun
anonfunorg
a
p
a
c
h
e
apache
apachespark
s
q
l
sql
sqlhive
e
x
e
c
u
t
i
o
n
execution
executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1$1.apply(Inser…anonfun
o
r
g
org
orgapache
s
p
a
r
k
spark
sparksql
h
i
v
e
hive
hiveexecution
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
InsertIntoHiveTable
InsertIntoHiveTable$writeToFile$1
1.
a
p
p
l
y
(
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
c
a
l
a
:
104
)
a
t
s
c
a
l
a
.
c
o
l
l
e
c
t
i
o
n
.
I
t
e
r
a
t
o
r
1.apply(InsertIntoHiveTable.scala:104) at scala.collection.Iterator
1.apply(InsertIntoHiveTable.scala:104)atscala.collection.Iteratorclass.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org
a
p
a
c
h
e
apache
apachespark
s
q
l
sql
sqlhive
e
x
e
c
u
t
i
o
n
execution
executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1(InsertIntoHiv…anonfun$saveAsHiveFile
3.
a
p
p
l
y
(
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
c
a
l
a
:
84
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
h
i
v
e
.
e
x
e
c
u
t
i
o
n
.
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable
3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable
a
n
o
n
f
u
n
anonfun
anonfunsaveAsHiveFile
3.
a
p
p
l
y
(
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
c
a
l
a
:
84
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
R
e
s
u
l
t
T
a
s
k
.
r
u
n
T
a
s
k
(
R
e
s
u
l
t
T
a
s
k
.
s
c
a
l
a
:
66
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
T
a
s
k
.
r
u
n
(
T
a
s
k
.
s
c
a
l
a
:
89
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
e
x
e
c
u
t
o
r
.
E
x
e
c
u
t
o
r
3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor
3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)atorg.apache.spark.scheduler.Task.run(Task.scala:89)atorg.apache.spark.executor.ExecutorTaskRunner.run(Executor.scala:227)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org
a
p
a
c
h
e
apache
apachespark
s
c
h
e
d
u
l
e
r
scheduler
schedulerDAGScheduler
f
a
i
l
J
o
b
A
n
d
I
n
d
e
p
e
n
d
e
n
t
S
t
a
g
e
s
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
1431
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
D
A
G
S
c
h
e
d
u
l
e
r
failJobAndIndependentStages(DAGScheduler.scala:1431) at org.apache.spark.scheduler.DAGScheduler
failJobAndIndependentStages(DAGScheduler.scala:1431)atorg.apache.spark.scheduler.DAGScheduleranonfun$abortStage
1.
a
p
p
l
y
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
1419
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
D
A
G
S
c
h
e
d
u
l
e
r
1.apply(DAGScheduler.scala:1419) at org.apache.spark.scheduler.DAGScheduler
1.apply(DAGScheduler.scala:1419)atorg.apache.spark.scheduler.DAGScheduler
a
n
o
n
f
u
n
anonfun
anonfunabortStage
1.
a
p
p
l
y
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
1418
)
a
t
s
c
a
l
a
.
c
o
l
l
e
c
t
i
o
n
.
m
u
t
a
b
l
e
.
R
e
s
i
z
a
b
l
e
A
r
r
a
y
1.apply(DAGScheduler.scala:1418) at scala.collection.mutable.ResizableArray
1.apply(DAGScheduler.scala:1418)atscala.collection.mutable.ResizableArrayclass.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1418)
at org.apache.spark.scheduler.DAGSchedulerKaTeX parse error: Can't use function '$' in math mode at position 8: anonfun$̲handleTaskSetFa…anonfun$handleTaskSetFailed
1.
a
p
p
l
y
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
799
)
a
t
s
c
a
l
a
.
O
p
t
i
o
n
.
f
o
r
e
a
c
h
(
O
p
t
i
o
n
.
s
c
a
l
a
:
236
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
D
A
G
S
c
h
e
d
u
l
e
r
.
h
a
n
d
l
e
T
a
s
k
S
e
t
F
a
i
l
e
d
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
799
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
D
A
G
S
c
h
e
d
u
l
e
r
E
v
e
n
t
P
r
o
c
e
s
s
L
o
o
p
.
d
o
O
n
R
e
c
e
i
v
e
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
1640
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
D
A
G
S
c
h
e
d
u
l
e
r
E
v
e
n
t
P
r
o
c
e
s
s
L
o
o
p
.
o
n
R
e
c
e
i
v
e
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
1599
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
D
A
G
S
c
h
e
d
u
l
e
r
E
v
e
n
t
P
r
o
c
e
s
s
L
o
o
p
.
o
n
R
e
c
e
i
v
e
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
1588
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
u
t
i
l
.
E
v
e
n
t
L
o
o
p
1.apply(DAGScheduler.scala:799) at scala.Option.foreach(Option.scala:236) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588) at org.apache.spark.util.EventLoop
1.apply(DAGScheduler.scala:799)atscala.Option.foreach(Option.scala:236)atorg.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799)atorg.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640)atorg.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599)atorg.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588)atorg.apache.spark.util.EventLoop$anon
1.
r
u
n
(
E
v
e
n
t
L
o
o
p
.
s
c
a
l
a
:
48
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
D
A
G
S
c
h
e
d
u
l
e
r
.
r
u
n
J
o
b
(
D
A
G
S
c
h
e
d
u
l
e
r
.
s
c
a
l
a
:
620
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
S
p
a
r
k
C
o
n
t
e
x
t
.
r
u
n
J
o
b
(
S
p
a
r
k
C
o
n
t
e
x
t
.
s
c
a
l
a
:
1832
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
S
p
a
r
k
C
o
n
t
e
x
t
.
r
u
n
J
o
b
(
S
p
a
r
k
C
o
n
t
e
x
t
.
s
c
a
l
a
:
1845
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
S
p
a
r
k
C
o
n
t
e
x
t
.
r
u
n
J
o
b
(
S
p
a
r
k
C
o
n
t
e
x
t
.
s
c
a
l
a
:
1922
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
h
i
v
e
.
e
x
e
c
u
t
i
o
n
.
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
a
v
e
A
s
H
i
v
e
F
i
l
e
(
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
c
a
l
a
:
84
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
h
i
v
e
.
e
x
e
c
u
t
i
o
n
.
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
i
d
e
E
f
f
e
c
t
R
e
s
u
l
t
1.run(EventLoop.scala:48) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1832) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1845) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1922) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.saveAsHiveFile(InsertIntoHiveTable.scala:84) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult
1.run(EventLoop.scala:48)atorg.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620)atorg.apache.spark.SparkContext.runJob(SparkContext.scala:1832)atorg.apache.spark.SparkContext.runJob(SparkContext.scala:1845)atorg.apache.spark.SparkContext.runJob(SparkContext.scala:1922)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable.saveAsHiveFile(InsertIntoHiveTable.scala:84)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResultlzycompute(InsertIntoHiveTable.scala:201)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult(InsertIntoHiveTable.scala:127)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.doExecute(InsertIntoHiveTable.scala:276)
at org.apache.spark.sql.execution.SparkPlanKaTeX parse error: Can't use function '$' in math mode at position 8: anonfun$̲execute$5.apply…anonfun$execute
5.
a
p
p
l
y
(
S
p
a
r
k
P
l
a
n
.
s
c
a
l
a
:
130
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
r
d
d
.
R
D
D
O
p
e
r
a
t
i
o
n
S
c
o
p
e
5.apply(SparkPlan.scala:130) at org.apache.spark.rdd.RDDOperationScope
5.apply(SparkPlan.scala:130)atorg.apache.spark.rdd.RDDOperationScope.withScope(RDDOperationScope.scala:150)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:130)
at org.apache.spark.sql.execution.QueryExecution.toRdd
l
z
y
c
o
m
p
u
t
e
(
Q
u
e
r
y
E
x
e
c
u
t
i
o
n
.
s
c
a
l
a
:
55
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
e
x
e
c
u
t
i
o
n
.
Q
u
e
r
y
E
x
e
c
u
t
i
o
n
.
t
o
R
d
d
(
Q
u
e
r
y
E
x
e
c
u
t
i
o
n
.
s
c
a
l
a
:
55
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
D
a
t
a
F
r
a
m
e
.
<
i
n
i
t
>
(
D
a
t
a
F
r
a
m
e
.
s
c
a
l
a
:
145
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
D
a
t
a
F
r
a
m
e
.
<
i
n
i
t
>
(
D
a
t
a
F
r
a
m
e
.
s
c
a
l
a
:
130
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
D
a
t
a
F
r
a
m
e
lzycompute(QueryExecution.scala:55) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55) at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:145) at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:130) at org.apache.spark.sql.DataFrame
lzycompute(QueryExecution.scala:55)atorg.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55)atorg.apache.spark.sql.DataFrame.<init>(DataFrame.scala:145)atorg.apache.spark.sql.DataFrame.<init>(DataFrame.scala:130)atorg.apache.spark.sql.DataFrame.apply(DataFrame.scala:52)
at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:817)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
at py4j.Gateway.invoke(Gateway.java:259)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:209)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.fs.viewfs.NotInMountpointException: getDefaultReplication on empty path is invalid
at org.apache.hadoop.fs.viewfs.ViewFileSystem.getDefaultReplication(ViewFileSystem.java:593)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.getStream(WriterImpl.java:2103)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.flushStripe(WriterImpl.java:2120)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.checkMemory(WriterImpl.java:352)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.notifyWriters(MemoryManager.java:168)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.addedRow(MemoryManager.java:157)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.addRow(WriterImpl.java:2413)
at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
O
r
c
R
e
c
o
r
d
W
r
i
t
e
r
.
w
r
i
t
e
(
O
r
c
O
u
t
p
u
t
F
o
r
m
a
t
.
j
a
v
a
:
86
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
h
i
v
e
.
e
x
e
c
u
t
i
o
n
.
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
OrcRecordWriter.write(OrcOutputFormat.java:86) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable
OrcRecordWriter.write(OrcOutputFormat.java:86)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable
a
n
o
n
f
u
n
anonfun
anonfunorg
a
p
a
c
h
e
apache
apachespark
s
q
l
sql
sqlhive
e
x
e
c
u
t
i
o
n
execution
executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1$1.apply(Inser…anonfun
o
r
g
org
orgapache
s
p
a
r
k
spark
sparksql
h
i
v
e
hive
hiveexecution
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
InsertIntoHiveTable
InsertIntoHiveTable$writeToFile$1
1.
a
p
p
l
y
(
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
c
a
l
a
:
104
)
a
t
s
c
a
l
a
.
c
o
l
l
e
c
t
i
o
n
.
I
t
e
r
a
t
o
r
1.apply(InsertIntoHiveTable.scala:104) at scala.collection.Iterator
1.apply(InsertIntoHiveTable.scala:104)atscala.collection.Iteratorclass.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org
a
p
a
c
h
e
apache
apachespark
s
q
l
sql
sqlhive
e
x
e
c
u
t
i
o
n
execution
executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1(InsertIntoHiv…anonfun$saveAsHiveFile
3.
a
p
p
l
y
(
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
c
a
l
a
:
84
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
q
l
.
h
i
v
e
.
e
x
e
c
u
t
i
o
n
.
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable
3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable
a
n
o
n
f
u
n
anonfun
anonfunsaveAsHiveFile
3.
a
p
p
l
y
(
I
n
s
e
r
t
I
n
t
o
H
i
v
e
T
a
b
l
e
.
s
c
a
l
a
:
84
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
R
e
s
u
l
t
T
a
s
k
.
r
u
n
T
a
s
k
(
R
e
s
u
l
t
T
a
s
k
.
s
c
a
l
a
:
66
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
s
c
h
e
d
u
l
e
r
.
T
a
s
k
.
r
u
n
(
T
a
s
k
.
s
c
a
l
a
:
89
)
a
t
o
r
g
.
a
p
a
c
h
e
.
s
p
a
r
k
.
e
x
e
c
u
t
o
r
.
E
x
e
c
u
t
o
r
3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor
3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)atorg.apache.spark.scheduler.Task.run(Task.scala:89)atorg.apache.spark.executor.ExecutorTaskRunner.run(Executor.scala:227)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
… 1 more
2
INSERT OVERWRITE TABLE eb_name PARTITION (DAY_ID = '20190611')
SELECT itv_prod_type,
itv_prod_name,
remark FROM tb_name ;
Error: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable@2085fb29
at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:179)
at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:459)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
at org.apache.hadoop.mapred.YarnChild
2.
r
u
n
(
Y
a
r
n
C
h
i
l
d
.
j
a
v
a
:
164
)
a
t
j
a
v
a
.
s
e
c
u
r
i
t
y
.
A
c
c
e
s
s
C
o
n
t
r
o
l
l
e
r
.
d
o
P
r
i
v
i
l
e
g
e
d
(
N
a
t
i
v
e
M
e
t
h
o
d
)
a
t
j
a
v
a
x
.
s
e
c
u
r
i
t
y
.
a
u
t
h
.
S
u
b
j
e
c
t
.
d
o
A
s
(
S
u
b
j
e
c
t
.
j
a
v
a
:
415
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
s
e
c
u
r
i
t
y
.
U
s
e
r
G
r
o
u
p
I
n
f
o
r
m
a
t
i
o
n
.
d
o
A
s
(
U
s
e
r
G
r
o
u
p
I
n
f
o
r
m
a
t
i
o
n
.
j
a
v
a
:
1917
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
m
a
p
r
e
d
.
Y
a
r
n
C
h
i
l
d
.
m
a
i
n
(
Y
a
r
n
C
h
i
l
d
.
j
a
v
a
:
158
)
C
a
u
s
e
d
b
y
:
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
q
l
.
m
e
t
a
d
a
t
a
.
H
i
v
e
E
x
c
e
p
t
i
o
n
:
H
i
v
e
R
u
n
t
i
m
e
E
r
r
o
r
w
h
i
l
e
p
r
o
c
e
s
s
i
n
g
w
r
i
t
a
b
l
e
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
s
e
r
d
e
2.
c
o
l
u
m
n
a
r
.
B
y
t
e
s
R
e
f
A
r
r
a
y
W
r
i
t
a
b
l
e
@
2085
f
b
29
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
q
l
.
e
x
e
c
.
M
a
p
O
p
e
r
a
t
o
r
.
p
r
o
c
e
s
s
(
M
a
p
O
p
e
r
a
t
o
r
.
j
a
v
a
:
505
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
q
l
.
e
x
e
c
.
m
r
.
E
x
e
c
M
a
p
p
e
r
.
m
a
p
(
E
x
e
c
M
a
p
p
e
r
.
j
a
v
a
:
170
)
.
.
.
8
m
o
r
e
C
a
u
s
e
d
b
y
:
j
a
v
a
.
l
a
n
g
.
C
l
a
s
s
C
a
s
t
E
x
c
e
p
t
i
o
n
:
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
s
e
r
d
e
2.
c
o
l
u
m
n
a
r
.
B
y
t
e
s
R
e
f
A
r
r
a
y
W
r
i
t
a
b
l
e
c
a
n
n
o
t
b
e
c
a
s
t
t
o
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
i
o
.
B
i
n
a
r
y
C
o
m
p
a
r
a
b
l
e
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
s
e
r
d
e
2.
l
a
z
y
.
L
a
z
y
S
i
m
p
l
e
S
e
r
D
e
.
d
o
D
e
s
e
r
i
a
l
i
z
e
(
L
a
z
y
S
i
m
p
l
e
S
e
r
D
e
.
j
a
v
a
:
162
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
s
e
r
d
e
2.
A
b
s
t
r
a
c
t
E
n
c
o
d
i
n
g
A
w
a
r
e
S
e
r
D
e
.
d
e
s
e
r
i
a
l
i
z
e
(
A
b
s
t
r
a
c
t
E
n
c
o
d
i
n
g
A
w
a
r
e
S
e
r
D
e
.
j
a
v
a
:
76
)
a
t
o
r
g
.
a
p
a
c
h
e
.
h
a
d
o
o
p
.
h
i
v
e
.
q
l
.
e
x
e
c
.
M
a
p
O
p
e
r
a
t
o
r
2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable@2085fb29 at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:505) at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:170) ... 8 more Caused by: java.lang.ClassCastException: org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable cannot be cast to org.apache.hadoop.io.BinaryComparable at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.doDeserialize(LazySimpleSerDe.java:162) at org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.deserialize(AbstractEncodingAwareSerDe.java:76) at org.apache.hadoop.hive.ql.exec.MapOperator
2.run(YarnChild.java:164)atjava.security.AccessController.doPrivileged(NativeMethod)atjavax.security.auth.Subject.doAs(Subject.java:415)atorg.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917)atorg.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)Causedby:org.apache.hadoop.hive.ql.metadata.HiveException:HiveRuntimeErrorwhileprocessingwritableorg.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable@2085fb29atorg.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:505)atorg.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:170)...8moreCausedby:java.lang.ClassCastException:org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritablecannotbecasttoorg.apache.hadoop.io.BinaryComparableatorg.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.doDeserialize(LazySimpleSerDe.java:162)atorg.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.deserialize(AbstractEncodingAwareSerDe.java:76)atorg.apache.hadoop.hive.ql.exec.MapOperatorMapOpCtx.readRow(MapOperator.java:136)
at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.access$200(MapOperator.java:100)
at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:496)
… 9 more
3612
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
