Hive UDF根据经纬度获取百度POI数据
有关百度poi资料：
零基础掌握百度地图兴趣点获取POI爬虫（python语言爬取）（基础篇）
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.Text;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
public class UDF_lnglat extends UDF {
public Text evaluate (String lat, String lng){
String s="";
if (null==lat || null==lng){
return null;
}
try {
s=getLocationByBaiduMap(lng, lat);
} catch (Exception e) {
e.printStackTrace();
}
return new Text(s);
}
public static void main(String[] args) {
String lat="31.931";
String lng="120.961";
Text evaluate = new UDF_lnglat().evaluate(lat,lng);
System.out.println(evaluate.toString());
}
public static String getLocationByBaiduMap(String longitude,String latitude) throws Exception {
String ak = "**自己的ak码*";
String locJson = geturl("http://api.map.baidu.com/geoconv/v1/?coords=" + longitude + "," +latitude + "&from=1&to=5&ak=" + ak);
System.out.println(locJson);
JSONObject jobject = JSON.parseObject(locJson);
JSONArray jsonArray = jobject.getJSONArray("result");
String lat=jsonArray.getJSONObject(0).getString("y");
String lng=jsonArray.getJSONObject(0).getString("x");
//System.out.println(lat);
String addrJson = geturl("http://api.map.baidu.com/geocoder/v2/?ak="+ ak +"&location=" + lat + "," + lng + "&output=json&pois=1");
System.out.println(addrJson);
JSONObject jobjectaddr = JSON.parseObject(addrJson);
JSONObject rJsonObject = jobjectaddr.getJSONObject("result");
System.out.println(rJsonObject);
String addr=jobjectaddr.getJSONObject("result").getString("formatted_address");
return addr;
}
private static String geturl(String geturl) throws Exception {
//请求的webservice的url
URL url = new URL(geturl);
//创建http链接,得到connection对象
HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
//设置请求的方法类型
httpURLConnection.setRequestMethod("POST");
//设置请求的内容类型
httpURLConnection.setRequestProperty("Content-type", "application/x-www-form-urlencoded");
//设置发送数据
httpURLConnection.setDoOutput(true);
//设置接受数据
httpURLConnection.setDoInput(true);
//发送数据,使用输出流
OutputStream outputStream = httpURLConnection.getOutputStream();
//发送的soap协议的数据
String content = "user_id="+ URLEncoder.encode("用户Id", "utf-8");
//发送数据
outputStream.write(content.getBytes());
//接收数据
InputStream inputStream = httpURLConnection.getInputStream();
BufferedReader in = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
StringBuffer buffer = new StringBuffer();
String line = "";
while ((line = in.readLine()) != null){
buffer.append(line);
}
String str = buffer.toString();
return str;
}
}
输出结果：
{"status":0,"result":[{"x":120.97172937368814,"y":31.935271870878876}]}
{"status":0,"result":{"location":{"lng":120.97172937368807,"lat":31.935271868820668},"formatted_address":"江苏省南通市通州区复兴东路","business":"世茂广场","addressComponent":{"country":"中国","country_code":0,"country_code_iso":"CHN","country_code_iso2":"CN","province":"江苏省","city":"南通市","city_level":2,"district":"通州区","town":"","town_code":"","adcode":"320612","street":"复兴东路","street_number":"","direction":"","distance":""},"pois":[{"addr":"复兴东路","cp":" ","direction":"东北","distance":"235","name":"雅居乐花园","poiType":"房地产","point":{"x":120.97089537994279,"y":31.933615356379954},"tag":"房地产;住宅区","tel":"","uid":"f854814c187262b9b35c6abe","zip":"","parent_poi":{"name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":"","distance":"","uid":""}},{"addr":"南通开发区复兴路与通盛大道交汇处东北角","cp":" ","direction":"东","distance":"363","name":"云樾东方","poiType":"房地产","point":{"x":120.96846097201075,"y":31.935254793744627},"tag":"房地产;住宅区","tel":"","uid":"0d1053d1073c7e4e882fb591","zip":"","parent_poi":{"name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":"","distance":"","uid":""}},{"addr":"南通市崇川区复兴东路与龙腾路交叉口东北150米","cp":" ","direction":"西","distance":"786","name":"万科·翡翠公园","poiType":"房地产","point":{"x":120.9787914853755,"y":31.935239472036778},"tag":"房地产;住宅区","tel":"","uid":"25a7c622389b58d982288ab5","zip":"","parent_poi":{"name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":"","distance":"","uid":""}},{"addr":"江苏省南通市通州区复兴东路与龙腾路交汇处东南侧","cp":" ","direction":"西","distance":"754","name":"绿地新里城","poiType":"房地产","point":{"x":120.9783962309511,"y":31.934243555494687},"tag":"房地产;住宅区","tel":"","uid":"0facb4fc809a24829222e0f5","zip":"","parent_poi":{"name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":"","distance":"","uid":""}},{"addr":"江苏省南通市崇川区星湖大道1066号","cp":" ","direction":"东北","distance":"811","name":"永旺梦乐城","poiType":"购物","point":{"x":120.96840707368014,"y":31.92973881228213},"tag":"购物;购物中心","tel":"","uid":"8cb8c30bf20a77139e9d68e8","zip":"","parent_poi":{"name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":"","distance":"","uid":""}},{"addr":"开发区能达CBD长虹路6号(携程呼叫中心北侧、复兴路口)","cp":" ","direction":"东","distance":"858","name":"鑫湖国贸中心-1号楼","poiType":"房地产","point":{"x":120.96417605472814,"y":31.933960100333267},"tag":"房地产;写字楼","tel":"","uid":"efd5b83daae581bb7adbd259","zip":"","parent_poi":{"name":"鑫湖国贸中心","tag":"房地产;住宅区","addr":"南通市长虹路6号","point":{"x":120.96376283419356,"y":31.934572975249098},"direction":"东","distance":"891","uid":"dcce022a620aab05c4f50d4b"}}],"roads":[],"poiRegions":[],"sematic_description":"雅居乐花园东北235米","cityCode":161}}
{"formatted_address":"江苏省南通市通州区复兴东路","business":"世茂广场","sematic_description":"雅居乐花园东北235米","cityCode":161,"roads":[],"location":{"lng":120.97172937368807,"lat":31.935271868820668},"poiRegions":[],"pois":[{"zip":"","uid":"f854814c187262b9b35c6abe","distance":"235","poiType":"房地产","name":"雅居乐花园","tel":"","parent_poi":{"uid":"","distance":"","name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":""},"tag":"房地产;住宅区","addr":"复兴东路","cp":" ","point":{"x":120.97089537994279,"y":31.933615356379954},"direction":"东北"},{"zip":"","uid":"0d1053d1073c7e4e882fb591","distance":"363","poiType":"房地产","name":"云樾东方","tel":"","parent_poi":{"uid":"","distance":"","name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":""},"tag":"房地产;住宅区","addr":"南通开发区复兴路与通盛大道交汇处东北角","cp":" ","point":{"x":120.96846097201075,"y":31.935254793744627},"direction":"东"},{"zip":"","uid":"25a7c622389b58d982288ab5","distance":"786","poiType":"房地产","name":"万科·翡翠公园","tel":"","parent_poi":{"uid":"","distance":"","name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":""},"tag":"房地产;住宅区","addr":"南通市崇川区复兴东路与龙腾路交叉口东北150米","cp":" ","point":{"x":120.9787914853755,"y":31.935239472036778},"direction":"西"},{"zip":"","uid":"0facb4fc809a24829222e0f5","distance":"754","poiType":"房地产","name":"绿地新里城","tel":"","parent_poi":{"uid":"","distance":"","name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":""},"tag":"房地产;住宅区","addr":"江苏省南通市通州区复兴东路与龙腾路交汇处东南侧","cp":" ","point":{"x":120.9783962309511,"y":31.934243555494687},"direction":"西"},{"zip":"","uid":"8cb8c30bf20a77139e9d68e8","distance":"811","poiType":"购物","name":"永旺梦乐城","tel":"","parent_poi":{"uid":"","distance":"","name":"","tag":"","addr":"","point":{"x":0.0,"y":0.0},"direction":""},"tag":"购物;购物中心","addr":"江苏省南通市崇川区星湖大道1066号","cp":" ","point":{"x":120.96840707368014,"y":31.92973881228213},"direction":"东北"},{"zip":"","uid":"efd5b83daae581bb7adbd259","distance":"858","poiType":"房地产","name":"鑫湖国贸中心-1号楼","tel":"","parent_poi":{"uid":"dcce022a620aab05c4f50d4b","distance":"891","name":"鑫湖国贸中心","tag":"房地产;住宅区","addr":"南通市长虹路6号","point":{"x":120.96376283419356,"y":31.934572975249098},"direction":"东"},"tag":"房地产;写字楼","addr":"开发区能达CBD长虹路6号(携程呼叫中心北侧、复兴路口)","cp":" ","point":{"x":120.96417605472814,"y":31.933960100333267},"direction":"东"}],"addressComponent":{"city_level":2,"country":"中国","town":"","distance":"","city":"南通市","adcode":"320612","country_code_iso":"CHN","country_code_iso2":"CN","country_code":0,"town_code":"","province":"江苏省","street":"复兴东路","district":"通州区","street_number":"","direction":""}}
江苏省南通市通州区复兴东路
{
"formatted_address": "江苏省南通市通州区复兴东路",
"business": "世茂广场",
"sematic_description": "雅居乐花园东北235米",
"cityCode": 161,
"roads": [ ],
"location": {
"lng": 120.97172937368806,
"lat": 31.935271868820667
},
"poiRegions": [ ],
"pois": [
{
"zip": "",
"uid": "f854814c187262b9b35c6abe",
"distance": "235",
"poiType": "房地产",
"name": "雅居乐花园",
"tel": "",
"parent_poi": {
"uid": "",
"distance": "",
"name": "",
"tag": "",
"addr": "",
"point": {
"x": 0,
"y": 0
},
"direction": ""
},
"tag": "房地产;住宅区",
"addr": "复兴东路",
"cp": " ",
"point": {
"x": 120.97089537994279,
"y": 31.933615356379953
},
"direction": "东北"
},
{
"zip": "",
"uid": "0d1053d1073c7e4e882fb591",
"distance": "363",
"poiType": "房地产",
"name": "云樾东方",
"tel": "",
"parent_poi": {
"uid": "",
"distance": "",
"name": "",
"tag": "",
"addr": "",
"point": {
"x": 0,
"y": 0
},
"direction": ""
},
"tag": "房地产;住宅区",
"addr": "南通开发区复兴路与通盛大道交汇处东北角",
"cp": " ",
"point": {
"x": 120.96846097201075,
"y": 31.935254793744626
},
"direction": "东"
},
{},
{},
{},
{}
],
"addressComponent": {
"city_level": 2,
"country": "中国",
"town": "",
"distance": "",
"city": "南通市",
"adcode": "320612",
"country_code_iso": "CHN",
"country_code_iso2": "CN",
"country_code": 0,
"town_code": "",
"province": "江苏省",
"street": "复兴东路",
"district": "通州区",
"street_number": "",
"direction": ""
}
}pom文件
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.xdb.udfdemo</groupId>
<artifactId>UDFDemo</artifactId>
<version>1.0-SNAPSHOT</version>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>1.1.0-cdh5.14.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.28</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*/RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
3360
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
