post工具_Get-Post-Json的网上爬虫

最新推荐文章于 2022-12-03 04:11:43 发布

weixin_39581318

最新推荐文章于 2022-12-03 04:11:43 发布

阅读量96

点赞数

文章标签： post工具

HttpClient ：

HttpClient 简介：

HttpClient 是 Apache Jakarta Common 下的子项目，可以用来提供高效的、最新的、 功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。 
HTTP 协议可能是现在 Internet 上使用得最多、最重要的协议了，越来越多的 Java 应用程序需要直接通过 HTTP 协议来访问网络资源。
虽然在 JDK 的 java net 包中已经提供了访问 HTTP 协议的基本功能，但是对于大部分应用程序来说，JDK 库本身提供的功能还不够丰富和灵活。

项目：

创建一个普通的Maven -Jar项目：

pom.xml配置文件：

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.msk</groupId>
  <artifactId>Httpclient</artifactId>
  <version>1.0-SNAPSHOT</version>

  <dependencies>
    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpclient</artifactId>
      <version>4.5.2</version>
    </dependency>
      <dependency>
          <groupId>junit</groupId>
          <artifactId>junit</artifactId>
          <version>4.12</version>
          <scope>test</scope>
      </dependency>
  </dependencies>
</project>

创建一个包和测试类：

get请求单独一个项目就可以完成，因为可以爬网上的测试，post和json需要重新创建工程方便测试。

package com.msk.test;


import org.apache.http.HttpEntity;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.security.URIParameter;
import java.util.ArrayList;
import java.util.List;

public class HttpClientTest {
    public static void main(String[] args) throws Exception {
   /* HttpClientTest.TestCloseableHttpClient();*/
      //  HttpClientTest.dogetParam();
        //HttpClientTest.dopostTest();
            //HttpClientTest.dopostTestParam();
                HttpClientTest.dopostTestParamJson();
    }

    /**
     * 爬虫不带参数的get
     * @throws Exception
     */
  /*  public static void TestCloseableHttpClient()throws Exception {
        //创建连接对象
       CloseableHttpClient client= HttpClients.createDefault();
         //创建get连接
        HttpGet get=new HttpGet("http://www.baidu.com");
        //发送请求并返回响应
        CloseableHttpResponse response=client.execute(get);
        //处理响应
        //处理响应状态码
        int code=response.getStatusLine().getStatusCode();
        System.out.println(code);
        //获取响应内容
        HttpEntity entity=response.getEntity();
        //创建工具类去获取响应内容
        String contnet=EntityUtils.toString(entity,"utf-8");
        System.out.println(contnet);
        //关闭连接
        client.close();
    }*/
    /**
     * 带参数的get请求爬虫
     */
    /*public static void dogetParam()throws Exception{
        //创建连接
        CloseableHttpClient client= HttpClients.createDefault();
        //创建一个封装URL的对象
        URIBuilder bui=new URIBuilder("https://www.baidu.com");
        bui.addParameter("wd","西游记");
        //创建一个get对象
        HttpGet get=new HttpGet(bui.build());
        //发送请求并返回响应
        CloseableHttpResponse response=client.execute(get);
        //处理响应
        //处理响应状态码
        int code=response.getStatusLine().getStatusCode();
        System.out.println(code);
        //获取响应内容
        HttpEntity entity=response.getEntity();
        //创建工具类去获取响应内容
        String contnet=EntityUtils.toString(entity,"utf-8");
        System.out.println(contnet);
        //关闭连接
        client.close();
    }*/
    /**
     * 发送post请求不带参数
     */
   /* public static void dopostTest()throws Exception{
        //创建连接
        CloseableHttpClient client=HttpClients.createDefault();
        //创建post连接
        HttpPost post=new HttpPost("http://localhost:8080/test/post");
        CloseableHttpResponse response=client.execute(post);

        //处理响应
        //处理响应状态码
        int code=response.getStatusLine().getStatusCode();
        System.out.println(code);
        //获取响应内容
        HttpEntity entity=response.getEntity();
        //创建工具类去获取响应内容
        String contnet=EntityUtils.toString(entity,"utf-8");
        System.out.println(contnet);
        //关闭连接
        client.close();
    }*/

    /**
     * 发送post请求带参数
     */
   /*public static void dopostTestParam()throws Exception{
       CloseableHttpClient client=HttpClients.createDefault();
       HttpPost post=new HttpPost("http://localhost:8080/test/post/param");
       //给定参数
       List<BasicNameValuePair>list=new ArrayList<>();
       list.add(new BasicNameValuePair("name","莫淞凯"));
       list.add(new BasicNameValuePair("pwd","mosongkai"));
       //获取到的参数的格式转换
       StringEntity entity=new UrlEncodedFormEntity(list,"utf-8");
       //绑定参数
       post.setEntity(entity);

       CloseableHttpResponse response=client.execute(post);
       //处理响应
       //处理响应状态码
       int code=response.getStatusLine().getStatusCode();
       System.out.println(code);
       //获取响应内容
       HttpEntity en=response.getEntity();
       //创建工具类去获取响应内容
       String contnet=EntityUtils.toString(en,"utf-8");
       System.out.println(contnet);
       //关闭连接
       client.close();
   }*/

    /**
     * 发送请求带Json格式
     */
    public static void dopostTestParamJson()throws Exception{
        CloseableHttpClient client=HttpClients.createDefault();
        HttpPost post=new HttpPost("http://localhost:8080/test/post/param/json");
        String json="{"name":"莫淞凯","pwd":"mosongkai"}";
        StringEntity entity=new StringEntity(json, ContentType.APPLICATION_JSON);

        //绑定参数
        post.setEntity(entity);

        CloseableHttpResponse response=client.execute(post);
        //处理响应
        //处理响应状态码
        int code=response.getStatusLine().getStatusCode();
        System.out.println(code);
        //获取响应内容
        HttpEntity en=response.getEntity();
        //创建工具类去获取响应内容
        String contnet=EntityUtils.toString(en,"utf-8");
        System.out.println(contnet);
        //关闭连接
        client.close();
    }
}

post和json的方法：

一个pom项目父工程-如：post-parent

pom配置文件：

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.msk</groupId>
  <artifactId>post-parent</artifactId>
  <packaging>pom</packaging>
  <version>1.0-SNAPSHOT</version>
  <modules>
    <module>postService</module>
  </modules>

  <!-- 对依赖的jar包的版本统一进行定义 -->
  <properties>
    <solrj.version>4.10.3</solrj.version>
    <jedis.version>2.9.0</jedis.version>
    <junit.version>4.12</junit.version>
    <spring.version>4.1.3.RELEASE</spring.version>
    <mybatis.version>3.2.8</mybatis.version>
    <mybatis.spring.version>1.2.2</mybatis.spring.version>
    <mysql.version>5.1.32</mysql.version>
    <slf4j.version>1.6.4</slf4j.version>
    <druid.version>1.0.9</druid.version>
    <jstl.version>1.2</jstl.version>
    <servlet-api.version>2.5</servlet-api.version>
    <tomcat.version>2.2</tomcat.version>
    <jsp-api.version>2.0</jsp-api.version>
    <zkClient-version>0.10</zkClient-version>
    <dubbo-version>2.5.4</dubbo-version>
    <jackson.version>2.4.2</jackson.version>
    <commons-net.version>3.3</commons-net.version>
    <commons-fileupload.version>1.3.1</commons-fileupload.version>
  </properties>


  <!-- jar包的依赖注入 ，由于该工程是一个父工程，所以jar包在该pom文件中只是声明 -->
  <dependencyManagement>
    <dependencies>

      <dependency>
        <groupId>org.apache.solr</groupId>
        <artifactId>solr-solrj</artifactId>
        <version>${solrj.version}</version>
      </dependency>
      <dependency>
        <groupId>redis.clients</groupId>
        <artifactId>jedis</artifactId>
        <version>${jedis.version}</version>
      </dependency>
      <!-- 单元测试 -->
      <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>${junit.version}</version>
      </dependency>
      <!-- 日志处理 -->
      <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-log4j12</artifactId>
        <version>${slf4j.version}</version>
      </dependency>
      <!-- Mybatis -->
      <dependency>
        <groupId>org.mybatis</groupId>
        <artifactId>mybatis</artifactId>
        <version>${mybatis.version}</version>
      </dependency>
      <dependency>
        <groupId>org.mybatis</groupId>
        <artifactId>mybatis-spring</artifactId>
        <version>${mybatis.spring.version}</version>
      </dependency>
      <!-- MySql -->
      <dependency>
        <groupId>mysql</groupId>
        <artifactId>mysql-connector-java</artifactId>
        <version>${mysql.version}</version>
      </dependency>
      <!-- 连接池 -->
      <dependency>
        <groupId>com.alibaba</groupId>
        <artifactId>druid</artifactId>
        <version>${druid.version}</version>
      </dependency>
      <!-- Spring -->
      <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-context</artifactId>
        <version>${spring.version}</version>
      </dependency>
      <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-beans</artifactId>
        <version>${spring.version}</version>
      </dependency>
      <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-webmvc</artifactId>
        <version>${spring.version}</version>
      </dependency>
      <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-jdbc</artifactId>
        <version>${spring.version}</version>
      </dependency>
      <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-aspects</artifactId>
        <version>${spring.version}</version>
      </dependency>
      <!-- JSP相关 -->
      <dependency>
        <groupId>jstl</groupId>
        <artifactId>jstl</artifactId>
        <version>${jstl.version}</version>
      </dependency>
      <dependency>
        <groupId>javax.servlet</groupId>
        <artifactId>servlet-api</artifactId>
        <version>${servlet-api.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>javax.servlet</groupId>
        <artifactId>jsp-api</artifactId>
        <version>${jsp-api.version}</version>
        <scope>provided</scope>
      </dependency>
      <!-- 文件上传组件 -->
      <dependency>
        <groupId>commons-fileupload</groupId>
        <artifactId>commons-fileupload</artifactId>
        <version>${commons-fileupload.version}</version>
      </dependency>

      <dependency>
        <groupId>commons-net</groupId>
        <artifactId>commons-net</artifactId>
        <version>${commons-net.version}</version>
      </dependency>

      <!-- Jackson Json处理工具包 -->
      <dependency>
        <groupId>com.fasterxml.jackson.core</groupId>
        <artifactId>jackson-databind</artifactId>
        <version>${jackson.version}</version>
      </dependency>
    </dependencies>
  </dependencyManagement>

  <build>
    <resources>
      <resource>
        <directory>src/main/java</directory>
        <includes>
          <include>**/*.xml</include>
        </includes>
      </resource>
      <resource>
        <directory>src/main/resources</directory>
        <includes>
          <include>**/*.xml</include>
          <include>**/*.properties</include>
        </includes>
      </resource>
    </resources>
    <!-- tomcat插件，由于子项目不一定每个都是web项目，所以该插件只是声明，并未开启 -->
    <pluginManagement>
      <plugins>
        <!-- 配置Tomcat插件 -->
        <plugin>
          <groupId>org.apache.tomcat.maven</groupId>
          <artifactId>tomcat7-maven-plugin</artifactId>
          <version>${tomcat.version}</version>
        </plugin>
      </plugins>
    </pluginManagement>
  </build>
</project>

一个基于父工程的war项目-如：postService

pom.xml配置文件：

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>post-parent</artifactId>
        <groupId>com.msk</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>postService</artifactId>
    <packaging>war</packaging>

    <dependencies>
        <!-- Spring -->
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-context</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-beans</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-webmvc</artifactId>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <!-- 配置Tomcat插件 -->
            <plugin>
                <groupId>org.apache.tomcat.maven</groupId>
                <artifactId>tomcat7-maven-plugin</artifactId>
               <configuration>
                   <path>/</path>
                   <port>8080</port>
               </configuration>
            </plugin>
        </plugins>
    </build>
</project>

在定义个resources目录，里面存放springMVC.xml配置文件；

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"
   xmlns:context="http://www.springframework.org/schema/context"
   xmlns:mvc="http://www.springframework.org/schema/mvc"
   xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
        http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc-4.0.xsd
        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
      <!-- 包的扫描器主要是扫描@controller -->
      <context:component-scan base-package="com.msk.web.controller"/>  
      <!-- 注册两个新对象 主要是为了来处理springmvc中的其他anntation 如：@requestmapping -->   
      <mvc:annotation-driven/>
</beans>

然后是web.xml

<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xmlns="http://java.sun.com/xml/ns/javaee"
         xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
         id="WebApp_ID" version="3.0">

  <!-- 配置springmvc -->
  <servlet>
    <servlet-name>springmvc</servlet-name>
    <servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
    <init-param>
      <param-name>contextConfigLocation</param-name>
      <param-value>classpath:springmvc.xml</param-value>
    </init-param>
    <load-on-startup>1</load-on-startup>
  </servlet>
  <servlet-mapping>
    <servlet-name>springmvc</servlet-name>
    <url-pattern>/</url-pattern>
  </servlet-mapping>

  <filter>
    <filter-name>encoding</filter-name>
    <filter-class>org.springframework.web.filter.CharacterEncodingFilter</filter-class>
    <init-param>
      <param-name>encoding</param-name>
      <param-value>utf-8</param-value>
    </init-param>
  </filter>
  <filter-mapping>
    <filter-name>encoding</filter-name>
    <url-pattern>/*</url-pattern>
  </filter-mapping>


  <welcome-file-list>
    <welcome-file>index.html</welcome-file>
    <welcome-file>index.htm</welcome-file>
    <welcome-file>index.jsp</welcome-file>
    <welcome-file>default.html</welcome-file>
    <welcome-file>default.htm</welcome-file>
    <welcome-file>default.jsp</welcome-file>
  </welcome-file-list>
</web-app>

接下里测试：

创建两个包：

com.msk.pojo-里面有个实体bean：Users

package com.msk.pojo;

import java.io.Serializable;

public class Users implements Serializable {
    private String name;
    private String pwd;

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getPwd() {
        return pwd;
    }

    public void setPwd(String pwd) {
        this.pwd = pwd;
    }
}

com.msk.web.controller

下面有个后台控制端-PostDemo：

package com.msk.pojo;

import java.io.Serializable;

public class Users implements Serializable {
    private String name;
    private String pwd;

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getPwd() {
        return pwd;
    }

    public void setPwd(String pwd) {
        this.pwd = pwd;
    }
}

weixin_39581318

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
post工具_Get-Post-Json的网上爬虫

HttpClient ：HttpClient 简介：HttpClient 是 Apache Jakarta Common 下的子项目，可以用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。 HTTP 协议可能是现在 Internet 上使用得最多、最重要的协议了，越来越多的 Java 应用程序需要直接通过 HTTP 协议来访...
复制链接

扫一扫