java爬虫_从腾讯视频播放界面爬取视频并存到本地

源码如下：

package com.example.demo.test.db;

import org.apache.commons.lang.RandomStringUtils;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * 获取视频接口的json
 * @author Administrator
 *
 */
public class CatchVedio {
    //    Socket client = new Scoket();
    private URL url;
    private HttpURLConnection urlConnection;
    private int responseCode;
    private BufferedReader reader;
    private BufferedWriter writer;


    public static void main(String[] args) {
        CatchVedio cv = new CatchVedio();
        try {

            String[] VedioURL = cv.get_VedioURL();//接收
            for(String temp:VedioURL) {//temp是每一个视频的播放地址
                StringBuilder stringBuilder = new StringBuilder();
                String filename= RandomStringUtils.randomAlphanumeric(10);
                String s = stringBuilder.append("D:\\worm\\video").append(File.separator).append(filename).append(".mp4").toString();
                CatchIMG.getImg(cv.analyse(cv.get_Json(temp)),s);
                //cv.toDownloadURL(cv.analyse(cv.get_Json(temp)));//写出到文件
            }
        } catch (IOException e) {
            // TODO 自动生成的 catch 块
            e.printStackTrace();
        }finally {
            try {
                cv.reader.close();
                cv.writer.close();
            } catch (IOException e) {
                // TODO 自动生成的 catch 块
                e.printStackTrace();
            }
        }

    }

    void toDownloadURL(String real_url) throws IOException {//将对应下载地址输出到文件
        this.writer = new BufferedWriter(new FileWriter("D:/worm/downloadURL.txt",true));//定义追加方式写入的流
//        this.writer.append(real_url);
        this.writer.write(real_url+"\r\n");
        this.writer.flush();
    }

    String analyse(String json) {//分析json,传回完整下载地址
        int fvkey_index = json.indexOf("\"fvkey\":\"")+9;
        int endIndex = json.indexOf("\"",fvkey_index);
        String fvkey = json.substring(fvkey_index,endIndex);//获取到fvkey
//        System.out.println(fvkey);

        int fn_index = json.indexOf("\"fn\":\"")+6;
        int fn_end = json.indexOf("\"",fn_index);
        String fn = json.substring(fn_index,fn_end);//获取到视频文件名
//        System.out.println(fn);

        String head = "http://ugcws.video.gtimg.com/";

        StringBuffer real_url = new StringBuffer();
        real_url.append(head);//加入头部
        real_url.append(fn+"?");//加入文件名
        real_url.append("vkey="+fvkey);//加入解锁码
        /*构造成功*/
//        System.out.println(real_url.toString());
        return real_url.toString();

    }

    String get_Json(String url) throws UnsupportedEncodingException, IOException {
        String line = "";
        StringBuffer sb = new StringBuffer();
        this.url = new URL(url);
        this.urlConnection = (HttpURLConnection)this.url.openConnection();
        this.responseCode = this.urlConnection.getResponseCode();
        if (this.responseCode == 200) {
            this.reader = new BufferedReader(new InputStreamReader(this.urlConnection.getInputStream(), "UTF-8"));
            while ((line = this.reader.readLine()) != null) {
                sb.append(line);// 网页传回的只有一行
            }
            return sb.toString();
        }
        return "";
    }

    String[] get_VedioURL() throws IOException {
//    void get_VedioURL() throws IOException {
        File file = new File("D:/worm/vedioURL.txt");
        String line = "";
        this.reader = new BufferedReader(new FileReader(file));
        String[] t = new String[0];
        List<String> container = new ArrayList<String>();
        while(null!=(line = this.reader.readLine())) {
            if(line.equals("")) {
                continue;
            }
            line = this.change(line);//转换一下
            container.add(line);//装入容器
        }
        return container.toArray(t);
    }
    /**
     * http://vv.video.qq.com/getinfo?vids=x0164ytbgov&platform=101001&charge=0&otype=json&defn=shd //格式
     * @param str
     * @return
     * https://v.qq.com/x/page/f08302y6rof.html//页面地址示例
     * https://v.qq.com/x/page/y083158hphd.html
     * https://v.qq.com/x/page/c08503oe58c.html
     */
    String change(String str) {//定义从页面播放地址获取vid转换到后台接口地址的方法
        String head = "http://vv.video.qq.com/getinfo?vids=";
        String tail = "&platform=101001&charge=0&otype=json&defn=shd";
        String vid = str.substring(str.indexOf("page/")+5,str.indexOf(".html"));
        return head+vid+tail;
    }
}

package com.example.demo.test.db;

import org.apache.commons.lang.RandomStringUtils;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;

/**
 * @ClassName: CatchIMG
 * @Description: 爬取一个指定地址的网络資源
 * @author penny
 * @date 2017年12月3日 下午9:00:05
 *
 */
public class CatchIMG {

    /**
     *
     * @Title: getImg
     * @Description: 通过一个url 去获取資源
     * @param @param url 连接地址
     * @param @throws IOException
     * @throws
     */
    public static void getImg(String url, String img) throws IOException{
        long startTime = System.currentTimeMillis();
        URL imgURL = new URL(url.trim());//转换URL
        HttpURLConnection urlConn = (HttpURLConnection) imgURL.openConnection();//构造连接
        urlConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36");
        urlConn.connect();
        System.out.println(CatchIMG.class.toString()+":获取连接="+urlConn.getResponseMessage());
        if(urlConn.getResponseCode()==200){//返回的状态码是200 表示成功
            InputStream ins = urlConn.getInputStream(); //获取输入流,从网站读取数据到 内存中
            OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(img)));
            int len=0;
            byte[] buff = new byte[1024*10];//10k缓冲流 视你内存大小而定咯

            while(-1!=(len=(new BufferedInputStream(ins)).read(buff))){//长度保存到len,内容放入到 buff
                out.write(buff, 0, len);//将图片数组内容写入到图片文件
//              System.out.println(CatchIMG.class.toString()+":"+len+"byte已经写入到文件中，内容:  "+new String(buff));
            }
            urlConn.disconnect();
            ins.close();
            out.close();
            //System.out.println(CatchIMG.class.toString()+"：获取图片完成,耗时="+((System.currentTimeMillis()-startTime)/1000)+"s");
        }
    }
    /**
     * @throws IOException
     * @Title: main
     * @Description: 测试方法
     * @throws
     */
//    public static void main(String[] args) throws IOException {
//        //文件名称
//        StringBuilder stringBuilder = new StringBuilder();
//        String filename= RandomStringUtils.randomAlphanumeric(10);
//        String s = stringBuilder.append("F:\\img").append(File.separator).append(filename).append(".avi").toString();
//        CatchIMG.getImg("https://www.imooc.com/video/20607/",s);
//    }

}