使用spark对dataframe的某一列调用HTTP请求并返回数据给dataframe

最新推荐文章于 2024-08-14 11:12:16 发布

Alex.liu

最新推荐文章于 2024-08-14 11:12:16 发布

阅读量5.9k

点赞数 2

分类专栏： Spark 文章标签： spark scala datafrmae HTTP请求

本文链接：https://blog.csdn.net/longwei92/article/details/93379585

版权

本文介绍如何在Spark DataFrame操作中针对某一列执行HTTP请求，并将返回的数据整合到DataFrame中。通过示例代码和Maven配置，详细阐述这一过程。

摘要由CSDN通过智能技术生成

代码

package cn.spark.sql

import java.net.URLEncoder
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.spark.sql.SparkSession
import scala.io.Source
import org.apache.spark.sql.functions._

object HttpForDataFrame{
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName("HttpForDataFrame")
      .master("local")
      .getOrCreate()
    spark.sparkContext.setLogLevel("ERROR")

    import spark.implicits._

    //获取datafrme数据集
    val studentScoresDF = spark.read.json("E:\\SparkStudy\\testData\\student.json").repation(n)
    studentScoresDF.show()


    //http请求代码
    def send_http_request(kw: String): String = {
      //测试用url，返回值为json格式的字符串，url变量需用URLEncoder.encode(string, "UTF-8")进行转码，防止出现index编码错误
      val url = "http://baike.baidu.com/api/openapi/BaikeLemmaCardApi?scope="+URLEncoder.encode("103", "UTF-8")+"&format="+URLEncoder.encode("json", "UTF-8")+"&appid="+URLEncoder.encode("379020", "UTF-8")+"&bk_key="+URLEncoder.encode(kw, "UTF-8")+"&bk_length="+URLEncoder.encode("600", "UTF-8")
      println(url)

      val httpclient = new DefaultHttpClient()
      try {
        val response = httpclient.execute(new HttpGet(url))
        val entity = response.getEntity
        val result = Source.fromInputStream(entity.getContent).getLines().mkString("\n")
        //println(result)
        return result
      } catch {
        //case ex: Excep