java爬虫程序 下载_JAVA网络爬虫批量下载豌豆夹中的应用

packagestudyjava;importjava.io.BufferedReader;importjava.io.File;importjava.io.FileOutputStream;importjava.io.FileWriter;importjava.io.IOException;importjava.io.InputStream;importjava.io.InputStreamReader;importjava.net.HttpURLConnection;importjava.net.MalformedURLException;importjava.net.URL;importjava.net.URLConnection;importjava.util.regex.Matcher;importjava.util.regex.Pattern;importjava.util.stream.Stream;importjava.net.HttpURLConnection;importjavax.xml.transform.stream.StreamSource;public classclient{public static void main(String[] args) throwsIOException

{//在D盘创建两份文件，一份为获取到的所有APK下载链接，一份为下载的所有应用名称

File f1=new File("D:\\down.txt");

File f2=new File("D:\\down_appname.txt");if(f1.exists())

f1.delete();elsef1.createNewFile();if(f2.exists())

f2.delete();elsef2.createNewFile();

FileWriter fw=new FileWriter(f1,true);

FileWriter fw2=new FileWriter(f2,true);

String str;//下载的网址，如豌豆夹必备应用页

URL url=new URL("http://www.wandoujia.com/essential/app");//下面就是给定一个网址，你怎么去获取它的源码咯

URLConnection conn=url.openConnection();

BufferedReader br=new BufferedReader(newInputStreamReader(conn.getInputStream()));//下面是争对每一行输入流进行过滤，是网址的给我弄出来，然后再创建下载链接

while((str=br.readLine())!=null)

{

Pattern pattern=Pattern.compile("http://apps.wandoujia.com/apps/.+.([a-z]+\\.)*.+/download\"\\sdownload=\".+\\s*.*\\.apk\"");

Matcher matcher=pattern.matcher(str);if(matcher.find())

{//正式写入下载网址和应用名称到相应文件中

String downurl=matcher.group().split("\\s")[0].split("\"")[0];

String downname=matcher.group().split("=")[1];

fw.write(downurl);

fw.write("\n");

fw2.write(downname);

fw2.write("\n");//下面是下载应用了

byte[] buffer = new byte[4 * 1024];//获取应用的网络资源连接，

URL urldown=newURL(downurl);

HttpURLConnection conn2=(HttpURLConnection)urldown.openConnection();

conn2.setDoInput(true);

conn2.connect();//获取输入流

InputStream in=conn2.getInputStream();//创建放置下载应用的文件夹

File dir=new File("D:\\downapp");if(!dir.exists())

dir.mkdir();//创建下载的应用，文件名和存放路径

File appdown=new File(dir,downname.split("\"")[1]);if(!appdown.exists())

appdown.createNewFile();//获取输出流

FileOutputStream out=newFileOutputStream(appdown);int br1=-1;while((br1=(in.read(buffer)))!=-1)

{

out.write(buffer,0, br1);

}

out.flush();

in.close();

out.close();

}

}

fw.close();

fw2.close();

}

}