JAVA网络爬虫(2)-----带get或post请求参数

1.Get请求参数的

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.http.client.utils.*;

//get带参数请求
public class demo01 {
    public static void main(String[] args) throws Exception {
      //打开浏览器创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();
        //设置请求地址:http://www.itheima.com/search?keys=Java
		//创建URIBuilder连接
		URIBuilder uriBuilder =new URIBuilder("http://www.itheima.com/search");
		//设置参数,如果设置多个参数在直接.setParameter(key,value)直接加即可
		uriBuilder.setParameter("key","Java");
		//创建HttpGet对象,设置URL访问地址
        HttpGet httpGet=new HttpGet(uriBuilder.build());
		System.out.println("发送请求的信息"+httpGet);
        //按回车发起请求,返回响应,使用HttpClient对象发起请求
        CloseableHttpResponse response=null;
		try{
			response=httpClient.execute(httpGet);
			//j解析响应,
			if(response.getStatusLine().getStatusCode()==200){
				String content= EntityUtils.toString(response.getEntity(),"UTF-8");
				System.out.println(content.length());
			}
		}catch(Exception e){
			e.printStackTrace();
		}finally{
			//释放资源
			if(response==null){
				try{
					response.close();
					httpClient.close();//关闭浏览器
				}catch(Exception e){
					e.printStackTrace();
				}
				
			}
		}
    }
}

2.Post请求参数的

import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;

//Post带参数请求
public class demo02 {
    public static void main(String[] args) throws Exception {
      //打开浏览器创建HttpClient对象
        CloseableHttpClient httpClient= HttpClients.createDefault();
		//创建HttpPost请求,设置URL访问路径
		HttpPost httpPost=new HttpPost("http://www.itheima.com/search");
		//声明List集合,封装表单中的参数
		List<NameValuePair> params=new ArrayList<NameValuePair>();
		//设置请求地址为http://www.itheima.com/search?keys=Java
		//创建表单的Entity对象,第一个参数是封装好的表单数据,第二个是参数是编码
		UrlEncodedFormEntity formEntity=new UrlEncodedFormEntity(params,"utf-8");
		//设置表单的Entity对象到Post请求中
		httpPost.setEntity(formEntity);
        //按回车发起请求,返回响应,使用HttpClient对象发起请求
        CloseableHttpResponse response=null;
		try{
			response=httpClient.execute(httpPost);
			//j解析响应,
			if(response.getStatusLine().getStatusCode()==200){
				String content= EntityUtils.toString(response.getEntity(),"UTF-8");
				System.out.println(content.length());
			}
		}catch(Exception e){
			e.printStackTrace();
		}finally{
			//释放资源
			if(response==null){
				try{
					response.close();
					httpClient.close();//关闭浏览器
				}catch(Exception e){
					e.printStackTrace();
				}
				
			}
		}
    }
}

3.连接池请求

import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;

//使用连接池进行请求
public class demo03 {
    public static void main(String[] args) throws Exception {
        //创建连接池管理器
        PoolingHttpClientConnectionManager cm=new 
            PoolingHttpClientConnectionManager();
        //设置最大连接数
        cm.setMaxTotal(100);
        //设置每个主机的最大连接数
        cm.setDefaultMaxPerRoute(10);//让对于每个主机的连接数进行控制,防止多少去爬取一个主机
        //使用连接池管理器发起请求
        doGet(cm);
        doGet(cm);
    }
    public static void doGet(PoolingHttpClientConnectionManager cm){
        //b不是每次创建新的HttpClient 而是从连接池中获取httpClient对象
        CloseableHttpClient httpClient =HttpClients.custom().setConnectionManager(cm).build();
        HttpGet httpGet=new HttpGet("http://www.itcast.cn");

        CloseableHttpResponse response=null;
        try{
            response=httpClient.execute(httpGet);
            //j解析响应,
            if(response.getStatusLine().getStatusCode()==200){
                String content= EntityUtils.toString(response.getEntity(),"UTF-8");
                System.out.println(content.length());
            }
        }catch(Exception e){
            e.printStackTrace();
        }finally{
            //释放资源
            if(response==null){
                try{
                    response.close();
                    httpClient.close();//关闭浏览器
                }catch(Exception e){
                    e.printStackTrace();
                }
                
            }
        }
    }
}
 

4.请求参数

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.http.client.utils.*;
import org.apache.http.client.config.RequestConfig;

public class demo04 {
	public static void main(String[] args){

		CloseableHttpClient httpClient=HttpClients.createDefault();
		//创建HttpGet对象设置URL访问路径
		HttpGet httpGet=new HttpGet("http://www.itcast.cn");
		//配置请求信息
		RequestConfig config=RequestConfig.custom()
			.setConnectTimeout(1000)  ///创建连接的最长时间,单位是毫秒
			.setConnectionRequestTimeout(500) //创建获取连接的最长时间
			.setSocketTimeout(10*1000)   //设置数据传输的最长时间
			.build();
		httpGet.setConfig(config);
		CloseableHttpResponse response=null;
		try{
			response=httpClient.execute(httpGet);
			//解析响应
			if(response.getStatusLine().getStatusCode()==200){
				String content= EntityUtils.toString(response.getEntity(),"UTF-8");
				System.out.println(content.length());
			}
		}catch(Exception e){
			e.printStackTrace();
		}finally{
			//释放资源
			if(response==null){
				try{
					response.close();
					httpClient.close();//关闭浏览器
				}catch(Exception e){
					e.printStackTrace();
				}
				
			}
		}
    }
}