java.lang.IllegalArgumentException: Illegal pattern component: XXX
at org.apache.commons.lang3.time.FastDatePrinter.parsePattern(FastDatePrinter.java:282) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FastDatePrinter.init(FastDatePrinter.java:149) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FastDatePrinter.<init>(FastDatePrinter.java:142) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FastDateFormat.<init>(FastDateFormat.java:384) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FastDateFormat.<init>(FastDateFormat.java:369) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FastDateFormat$1.createInstance(FastDateFormat.java:91) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FastDateFormat$1.createInstance(FastDateFormat.java:88) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FormatCache.getInstance(FormatCache.java:82) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.commons.lang3.time.FastDateFormat.getInstance(FastDateFormat.java:165) ~[commons-lang3-3.3.2.jar:3.3.2]
at org.apache.spark.sql.execution.datasources.csv.CSVOptions.<init>(CSVOptions.scala:128) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.execution.datasources.csv.CSVOptions.<init>(CSVOptions.scala:39) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.execution.datasources.csv.CSVFileFormat.inferSchema(CSVFileFormat.scala:55) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$7.apply(DataSource.scala:177) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$7.apply(DataSource.scala:177) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at scala.Option.orElse(Option.scala:289) ~[scala-library-2.11.11.jar:na]
at org.apache.spark.sql.execution.datasources.DataSource.getOrInferFileFormatSchema(DataSource.scala:176) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:366) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:178) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:533) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:412) ~[spark-sql_2.11-2.2.0.jar:2.2.0]
at com.gw.spark.utils.HdfsFileUtils$.getCsvFileFromHdfs(HdfsFileUtils.scala:94) ~[Spark_M.jar:na]
at com.gw.spark.spark.MyAls.trainAndSaveResultByHdfs(MyAls.scala:82) ~[Spark_M.jar:na]
at com.matrix.utils.AlsTest.main(AlsTest.java:18) ~[classes/:na]
at com.matrix.service.impl.MtxUserRecommendServiceImpl.recommend(MtxUserRecommendServiceImpl.java:61) ~[classes/:na]
at com.matrix.service.impl.MtxUserRecommendServiceImpl$$FastClassBySpringCGLIB$$b10f73a0.invoke(<generated>) ~[spring-core-4.0.2.RELEASE.jar:na]
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) ~[spring-core-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:711) ~[spring-aop-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) ~[spring-aop-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:98) ~[spring-tx-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:262) ~[spring-tx-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:95) ~[spring-tx-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) ~[spring-aop-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) ~[spring-aop-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) ~[spring-aop-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:644) ~[spring-aop-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at com.matrix.service.impl.MtxUserRecommendServiceImpl$$EnhancerBySpringCGLIB$$c19b5bfa.recommend(<generated>) ~[spring-core-4.0.2.RELEASE.jar:na]
at com.matrix.api.TestController.t1(TestController.java:17) ~[classes/:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_66]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_66]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_66]
at java.lang.reflect.Method.invoke(Method.java:497) ~[na:1.8.0_66]
at org.springframework.web.method.support.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:215) ~[spring-web-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:132) ~[spring-web-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:104) ~[spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(RequestMappingHandlerAdapter.java:749) ~[spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:690) ~[spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:83) ~[spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:945) ~[spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:876) ~[spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:961) [spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:852) [spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:622) [servlet-api.jar:na]
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:837) [spring-webmvc-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:729) [servlet-api.jar:na]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:291) [catalina.jar:8.0.30]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) [catalina.jar:8.0.30]
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) [tomcat-websocket.jar:8.0.30]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:239) [catalina.jar:8.0.30]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) [catalina.jar:8.0.30]
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88) [spring-web-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:108) [spring-web-4.0.2.RELEASE.jar:4.0.2.RELEASE]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:239) [catalina.jar:8.0.30]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) [catalina.jar:8.0.30]
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:212) [catalina.jar:8.0.30]
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106) [catalina.jar:8.0.30]
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502) [catalina.jar:8.0.30]
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141) [catalina.jar:8.0.30]
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79) [catalina.jar:8.0.30]
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:616) [catalina.jar:8.0.30]
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88) [catalina.jar:8.0.30]
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:521) [catalina.jar:8.0.30]
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1096) [tomcat-coyote.jar:8.0.30]
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:674) [tomcat-coyote.jar:8.0.30]
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1500) [tomcat-coyote.jar:8.0.30]
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1456) [tomcat-coyote.jar:8.0.30]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_66]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_66]
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) [tomcat-util.jar:8.0.30]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_66]
/**
* 读取hdfs集群的文件
* @param sparkSession
* @param fileUrl hdfs中的路径
* @return
*/
def getCsvFileFromHdfs(sparkSession : SparkSession, fileUrl : String) : DataFrame = {
try{
return sparkSession.read.option("timestampFormat", "yyyy/MM/dd HH:mm:ss ZZ").csv(property.get("HADOOP_URI1").toString + fileUrl)
}catch {
case e : RemoteException =>
return sparkSession.read.option("timestampFormat", "yyyy/MM/dd HH:mm:ss ZZ").csv(property.get("HADOOP_URI2").toString + fileUrl)
}
}
/**
* 在hdfs上创建或者追加csv文件
* @param sparkSession
* @param dataFrame
* @param fileUrl
*/
def createOrAppendCsvFileFromHdfs(sparkSession: SparkSession, dataFrame : DataFrame, fileUrl : String) : Unit = {
try{
dataFrame.write.option("timestampFormat", "yyyy/MM/dd HH:mm:ss ZZ").mode(SaveMode.Append).csv(property.get("HADOOP_URI1").toString + fileUrl)
}catch {
case e : RemoteException =>
dataFrame.write.option("timestampFormat", "yyyy/MM/dd HH:mm:ss ZZ").mode(SaveMode.Append).csv(property.get("HADOOP_URI2").toString + fileUrl)
}
}
在直接使用java main函数使用读取spark读取hadoop中的csv没有问题
在spring中集成使用就回出现上述问题
解决方式:如上所示,直接在读取和写入的时候加入option,设置为ZZ,最后