java icon包_java批量采集豌豆荚网站Android应用图标和包名

package im.garth.AppIconDownloader;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileOutputStream;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.net.URL;

import java.net.URLConnection;

import java.util.HashMap;

import java.util.Map.Entry;

import org.apache.http.HttpEntity;

import org.apache.http.HttpResponse;

import org.apache.http.HttpStatus;

import org.apache.http.client.HttpClient;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.DefaultHttpClient;

import org.apache.http.util.EntityUtils;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

/**

* 获取豌豆荚网页上安卓软件全部软件

* 注意：执行程序前，一定要在这个工程目录下创建icon文件夹

* 所有图标将下载到icon这个文件夹中

* 应用名称与包名写到了icon下的packageName.txt文件里

*

* 这个程序用到的jar包：

* commons-logging-1.1.3.jar

* httpclient-4.1.2.jar

* httpcore-4.3.jar

* jsoup-1.6.1.jar

*

*

*/

public class AppIconDownloader {

/**

* @param args

*/

public static void main(String[] args) {

String rootUrl = "http://www.wandoujia.com/tag/全部软件/total?page=";

//String rootUrl = "http://www.wandoujia.com/tag/全部游戏/total?page=";

//下载1到20页的应用图标

for(int i = 1; i < = 20; i++) {

System.out.println("【下载进度】：准备下载第" + i + "页");

String currentUrl = rootUrl + i;

HashMap apps = new HashMap();

apps = getAppImageUrl(currentUrl);

//遍历HashMap逐个下载图标

for(Entry entry : apps.entrySet()) {

try{

//下载图标，存储到当前工程目录下的icon目录(请事先创建icon目录)

download(entry.getValue(), "icon/" + entry.getKey() + ".png");

}catch(Exception e) {

System.out.println("【下载出错】：" + entry.getKey());

e.printStackTrace();

}

}

System.out.println("【下载进度】：第" + i + "页下载完成");

}

}

/**

* 获取url网页里的所有应用的应用名及其图标网址

* @param appPackageName

* @return

*/

private static HashMap getAppImageUrl(String url) {

HashMap apps = new HashMap();

String appPackageName = "";

String appImageUrl = "";

String appName = "";

String html = getHtmlByUrl(url);

Document doc = Jsoup.parse(html);

Elements elements = doc.select("div.container.clearfix>section.main-col>div.app-blocks>div.app-block>ul.app-list.clearfix>li.app>a.icon-area");

Elements nameElements = doc.select("div.container.clearfix>section.main-col>div.app-blocks>div.app-block>ul.app-list.clearfix>li.app>div.operate>a.name>span.txt");

Elements imageEle;

int i = 0;

for(Element ele : elements) {

//获取包名

appPackageName = ele.attr("data-pn");

//获取图标网址

imageEle = ele.select("img.icon");

appImageUrl = imageEle.get(0).attr("src").replace("68_68", "256_256");

//加入apps

apps.put(appPackageName, appImageUrl);

//获取app名称

appName = nameElements.get(i).text();

//把app名称和包名输出到文件

write2file("【" + appName + "】" + appPackageName);

i++;

}

System.out.println("【下载进度】：" + url + "下的图标网址已经获取成功");

return apps;

}

/**

* 下载文件到本地

*

* @param urlString

*          被下载的文件地址

* @param filename

*          本地文件名

* @throws Exception

*           各种异常

*/

private static void download(String urlString, String filename) throws Exception {

System.out.println("【下载进度】：正在下载" + filename);

// 构造URL

URL url = new URL(urlString);

// 打开连接

URLConnection con = url.openConnection();

// 输入流

InputStream is = con.getInputStream();

// 1K的数据缓冲

byte[] bs = new byte[1024];

// 读取到的数据长度

int len;

// 输出的文件流

OutputStream os = new FileOutputStream(filename);

// 开始读取

while ((len = is.read(bs)) != -1) {

os.write(bs, 0, len);

}

// 完毕，关闭所有链接

os.close();

is.close();

System.out.println("【下载进度】：" + filename + "下载成功");

}

/**

* 根据URL获得所有的html信息

* @param url

* @return html

*/

private static String getHtmlByUrl(String url){

String html = null;

//创建httpClient对象

HttpClient httpClient = new DefaultHttpClient();

//以get方式请求该URL

HttpGet httpget = new HttpGet(url);

try {

//得到responce对象

HttpResponse responce = httpClient.execute(httpget);

//返回码

int resStatu = responce.getStatusLine().getStatusCode();

//200正常  其他就不对

if (resStatu==HttpStatus.SC_OK) {

//获得相应实体

HttpEntity entity = responce.getEntity();

if (entity!=null) {

//获得html源代码

html = EntityUtils.toString(entity);

}

}

} catch (Exception e) {

System.out.println("访问【"+url+"】出现异常!");

e.printStackTrace();

} finally {

httpClient.getConnectionManager().shutdown();

}

return html;

}

private static void write2file(String content) {

File file = new File("icon/packageName.txt");

BufferedWriter writer = null;

try{

if(!file.exists()) {

file.createNewFile();

}

//参数true表示将输出追加到文件内容的末尾而不覆盖原来的内容

writer = new BufferedWriter(new FileWriter(file, true));

//输出内容

writer.write(content);

//换行

writer.newLine();

} catch(IOException e){

System.out.println("输出出错");

e.printStackTrace();

} finally {

if( writer != null) {

try {

writer.close();

} catch(IOException e) {

e.printStackTrace();

}

}

}

}

}