proxy_gater proxy 教程-CSDN博客

本文链接：https://blog.csdn.net/gdhgr/article/details/90696769
package test;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.modules.junit4.PowerMockRunner;

import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Random;

//@RunWith(PowerMockRunner.class)
public class testspider {
    //http://market.finance.sina.com.cn/transHis.php?symbol=sz000001&date=2018-04-27&page=5
//    @Test
    public void test() throws IOException {
        InetAddress localHost = InetAddress.getLocalHost();
         InetAddress byName = InetAddress.getByName("http://market.finance.sina.com.cn");
//        InetAddress[] allByName = InetAddress.getAllByName("http://market.finance.sina.com.cn");
        String url  ="http://market.finance.sina.com.cn/transHis.php?symbol=sz000001&date=2018-04-27&page=5";

        //199.67.131.156
//        PowerMockito.when(InetAddress.getByName(Mockito.anyString())).thenReturn()
//        String url  ="http://www.baidu.com";
        Connection tempConn = Jsoup.connect(url);
        //模拟浏览器的请求头
        tempConn.header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0");
        //开始连接HTTP请求。
        Connection.Response demo = tempConn.ignoreContentType(true).method(Connection.Method.GET)
                .execute();
        Document documentDemo = demo.parse();
        //这里就是获取该页面的HTML元素。
        System.out.println(documentDemo.toString());
    }

    public static void main(String[] args) throws IOException {

        Random r = new Random();
        String[] ua = {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 OPR/37.0.2178.32",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586",
                "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
                "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
                "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
                "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.3 Safari/537.36",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.277.400 QQBrowser/9.4.7658.400",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7",
                "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0"};


//            InetAddress localHost = InetAddress.getLocalHost();
//            InetAddress byName = InetAddress.getByName("http://market.finance.sina.com.cn");
        InetAddress[] allByName = InetAddress.getAllByName("http://market.finance.sina.com.cn");
//            String url  ="http://market.finance.sina.com.cn/transHis.php?symbol=sz000001&date=2018-04-27&page=5";
//
//            
        PowerMockito.when(InetAddress.getByName(Mockito.anyString())).thenReturn()
        String url  ="http://www.baidu.com";
//            Connection tempConn = Jsoup.connect(url).proxy(");
//            //模拟浏览器的请求头
//            tempConn.header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0");
//            //开始连接HTTP请求。
//            Connection.Response demo = tempConn.ignoreContentType(true).method(Connection.Method.GET)
//                    .execute();
//            Document documentDemo = demo.parse();
//            //这里就是获取该页面的HTML元素。
//            System.out.println(documentDemo.toString());


        Document doc = Jsoup.connect("http://market.finance.sina.com.cn/transHis.php?symbol=sz000001&date=2018-04-27&page=5")
                .timeout(5000)
//                .proxy("h)
                .proxy(")
                .ignoreContentType(true)
                .userAgent(ua[r.nextInt(14)]).get();
        System.out.println(doc);
//                .header("referer","http://trace.yto.net.cn:8022/gw/index/index.html")//这个来源记得换..

    }

}