JAVA网络爬虫批量下载豌豆夹中的应用-CSDN博客

今天终于把脚本弄好了，虽然是东拼西凑的，总算有点成就感啦，下面把代码帖这～

package studyjava;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;

import java.net.HttpURLConnection;
import javax.xml.transform.stream.StreamSource;


public class client{
    public static void main(String[] args) throws IOException
    {   
        //在D盘创建两份文件，一份为获取到的所有APK下载链接，一份为下载的所有应用名称
        File f1=new File("D:\\down.txt");
        File f2=new File("D:\\down_appname.txt");
        if(f1.exists())
            f1.delete();
        else
            f1.createNewFile();
        
        if(f2.exists())
            f2.delete();
        else
            f2.createNewFile();
        
        FileWriter fw=new FileWriter(f1,true);
        FileWriter fw2=new FileWriter(f2,true);
        
        String str;
        //下载的网址，如豌豆夹必备应用页
        URL url=new URL("http://www.wandoujia.com/essential/app");
        //下面就是给定一个网址，你怎么去获取它的源码咯
        URLConnection conn=url.openConnection();
        BufferedReader br=new BufferedReader(new InputStreamReader(conn.getInputStream()));
        //下面是争对每一行输入流进行过滤，是网址的给我弄出来，然后再创建下载链接
        while((str=br.readLine())!=null)
        {   
            Pattern pattern=Pattern.compile("http://apps.wandoujia.com/apps/.+.([a-z]+\\.)*.+/download\"\\sdownload=\".+\\s*.*\\.apk\"");
            Matcher matcher=pattern.matcher(str);
            if(matcher.find())
            {   //正式写入下载网址和应用名称到相应文件中
                String downurl=matcher.group().split("\\s")[0].split("\"")[0];
                String downname=matcher.group().split("=")[1];
                fw.write(downurl);
                fw.write("\n");
                fw2.write(downname);
                fw2.write("\n");
                //下面是下载应用了
                byte[] buffer = new byte[4 * 1024];
                //获取应用的网络资源连接，
                URL urldown=new URL(downurl);
                HttpURLConnection conn2=(HttpURLConnection)urldown.openConnection();
                conn2.setDoInput(true);
                conn2.connect();
                //获取输入流
                InputStream in=conn2.getInputStream();
                //创建放置下载应用的文件夹
                File dir=new File("D:\\downapp");
                if(!dir.exists())
                    dir.mkdir();
                //创建下载的应用，文件名和存放路径
                File appdown=new File(dir,downname.split("\"")[1]);
                if(!appdown.exists())
                    appdown.createNewFile();
                //获取输出流
                FileOutputStream out=new FileOutputStream(appdown);
                
                int br1=-1;
                while((br1=(in.read(buffer)))!=-1)
                    {
                    out.write(buffer, 0, br1);                
                    }
                out.flush();
                in.close();
                out.close();

            }
        }
        fw.close();
        fw2.close();
}
}