![](https://img-blog.csdnimg.cn/20201014180756754.png?x-oss-process=image/resize,m_fixed,h_64,w_64)
spark
文章平均质量分 75
qq_38680817
这个作者很懒,什么都没留下…
展开
-
sparksql源码解析一:
parser调用sqlbaseParser的singleStatement方法,利用anltr4里面的singleStatement来目前主流的sql解析器有 anltr和calcite,如果使用选择使用anltr,SQL进行查询,首先需要将SQL解析成spark中的抽象语法树(AST)。在spark中是借助开源的antlr4库来解析的。Spark SQL的语法规则文件是:SqlBase.g4。在生成的文件中SqlBaseBaseListene原创 2021-12-07 10:09:21 · 1333 阅读 · 0 评论 -
多张报表简单逻辑在同一任务中样例
import java.text.SimpleDateFormatimport java.utilimport java.util.Dateimport com.alibaba.fastjson.JSONObjectimport com.fengtu.sparktest.utils.{JSONUtils, MD5Util, SparkUtils}import com.fengtu.sparktest.utils2.DateUt.原创 2021-08-02 10:26:26 · 182 阅读 · 0 评论 -
shell执行工具类
import org.slf4j.Logger;import org.slf4j.LoggerFactory;import java.io.IOException;public class ShellExcutor { private static Logger logger = LoggerFactory.getLogger(ShellExcutor.class); public static void e.原创 2021-08-02 10:19:15 · 146 阅读 · 0 评论 -
spark工具类(常用)
package com.fengtu.sparktest.utilsimport java.sql.DriverManagerimport java.util.{Map, Properties}import com.alibaba.fastjson.{JSONArray, JSONObject}import com.fengtu.sparktest.utils2.Utilsimport org.apache.log4j.Loggerimport org.apache.spark.rdd.R.原创 2021-08-02 10:18:11 · 510 阅读 · 0 评论 -
时间日期工具类
package com.fengtu.sparktest.utilsimport java.text.{ParseException, SimpleDateFormat}import java.util.{Calendar, Date}import scala.collection.mutable.ArrayBufferobject DateTimeUtil extends Serializable { def main(args: Array[String]): Unit = {.原创 2021-08-02 10:17:10 · 64 阅读 · 0 评论 -
spark多线程的案例
package com.fengtu.sparktest.aoiTestimport java.text.SimpleDateFormatimport java.util.Dateimport java.util.concurrent.{Callable, Executors, FutureTask}import com.alibaba.fastjson.{JSON, JSONArray, JSONObject}import com.fengtu.sparktest.aoi.GetGjAoi.原创 2021-08-02 10:15:08 · 470 阅读 · 0 评论 -
spark写入mysql业务案例
package com.fengtu.sparktest.aoiTestimport java.sql.Connectionimport java.security.MessageDigestimport com.alibaba.fastjson.JSONObjectimport com.fengtu.sparktest.utils.{DbUtils, JavaUtil, JdbcTemplateUtil, StringUtils}import org.apache.log4j.Logger.原创 2021-08-02 10:13:30 · 95 阅读 · 0 评论 -
spark业务案例
package com.fengtu.sparktest.aoiTestimport java.net.URLEncoderimport com.alibaba.fastjson.serializer.SerializerFeatureimport com.alibaba.fastjson.{JSON, JSONArray, JSONObject}import com.fengtu.sparktest.utils.SparkUtilsimport com.fengtu.sparktest.u.原创 2021-08-02 10:12:08 · 244 阅读 · 0 评论 -
spark复杂转换
package com.sf.gis.scala.base.sparkimport com.alibaba.fastjson.JSONObjectimport org.apache.log4j.Loggerimport org.apache.spark.rdd.RDDimport scala.collection.mutable.ArrayBufferimport scala.util.Random/** * Created by 01374443 on 2020/7/27. .原创 2021-08-02 10:10:47 · 88 阅读 · 0 评论 -
spark网络调优
package com.sf.gis.scala.base.sparkimport java.util.Calendarimport java.util.concurrent.{Executors, LinkedBlockingQueue}import java.util.concurrent.atomic.AtomicIntegerimport com.alibaba.fastjson.JSONObjectimport org.apache.log4j.Loggerimport org..原创 2021-08-02 10:10:16 · 128 阅读 · 0 评论 -
spark读工具类
package com.sf.gis.scala.base.sparkimport org.apache.log4j.Loggerimport org.apache.spark.rdd.RDDimport org.apache.spark.sql.{Row, SparkSession}import org.apache.spark.storage.StorageLevelimport com.alibaba.fastjson.JSONObjectimport com.sf.gis.java..原创 2021-08-02 10:09:37 · 131 阅读 · 0 评论 -
spark写入工具类
package com.sf.gis.scala.base.sparkimport java.util.Propertiesimport com.alibaba.fastjson.JSONObjectimport com.sf.gis.java.base.dto.DBInfoimport com.sf.gis.java.base.pojo.BasePojoimport com.sf.gis.java.base.util.{CalPartitionUtil, ObjectUtil}impor.原创 2021-08-02 10:09:01 · 104 阅读 · 0 评论 -
Spark初始化工具类
package com.sf.gis.scala.base.sparkimport org.apache.spark.SparkConfimport org.apache.spark.sql.SparkSessionimport org.apache.log4j.Logger/** * Created by 01374443 on 2020/7/27. */object Spark { @transient lazy val logger: Logger = Logger.get.原创 2021-08-02 10:07:13 · 196 阅读 · 1 评论 -
spark处理复杂情况的join
package com.sf.gis.scala.base.sparkimport org.apache.log4j.Loggerimport org.apache.spark.rdd.RDDimport scala.collection.mutable.ArrayBufferimport scala.util.Random/** * Created by 01374443 on 2020/7/27. * 处理一些复杂的关联场景 */object SparkJoin { .原创 2021-08-02 10:03:58 · 146 阅读 · 0 评论