spark 读 http 的 API 接口，将读出的JSON字符串，直接转成 DataFrame

普罗米修斯之火

已于 2023-08-11 11:24:56 修改

阅读量436

点赞数 1

分类专栏： spark 文章标签： spark http json

于 2023-08-11 10:04:24 首次发布

本文链接：https://blog.csdn.net/WuBoooo/article/details/132225055

版权

spark 专栏收录该内容

27 篇文章 0 订阅

订阅专栏

    import spark.implicits._

    val jsonObject = new JSONObject()
    jsonObject.fluentPut("startTime", "2023-08-10 00:00:00")
    jsonObject.fluentPut("endTime", "2023-08-10 10:00:00")
    jsonObject.fluentPut("pageNum", 1)
    jsonObject.fluentPut("pageSize", 500)

    val requestJson = HttpUtils.postRequest(Props.get("your url"), jsonObject.toString())
    val requestData = JSON.parseObject(requestJson).getString("data")
    val ds: Dataset[String] = spark.createDataset(Seq(requestData))
    val dataFrame = spark.read.json(ds)

    dataFrame.show(false)



  /**
   *
   * @param url url地址
   * @param bodyData 传入json字符串
   * @return request
   */
  def postRequest(url: String,bodyData: String): String = {
    val httpClient = HttpClientBuilder.create().build()
    val httpPost = new HttpPost(url)
    httpPost.setHeader("Content-Type", "application/json;charset=utf-8")
    //设置超时时间
    httpPost.setConfig(RequestConfig
      .custom()
      .setConnectTimeout(5000)
      .setConnectionRequestTimeout(2000)
      .setSocketTimeout(20000)
      .build())

    val entity = new StringEntity(bodyData,"utf-8")

    //代码修改
    entity.setChunked(false)

    httpPost.setEntity(entity)

    var response: CloseableHttpResponse = null
    var request: String = ""

    //设置SSL

    try {
      response = httpClient.execute(httpPost)
      request = EntityUtils.toString(response.getEntity,"UTF-8")
    } finally {
      try {
        httpClient.close()
        response.close()
      } catch {
        case e: IOException =>
          e.printStackTrace()
      }
    }
    request
  }