Java爬虫入门程序

今天刚开始看爬虫内容，跟着网上做了个Demo
先贴配置信息吧
log4j.properties

log4j.rootLogger=DEBUG,A1
log4j.logger.cn.itcast = DEBUG
###org.apache.log4j.ConsoleAppender（控制台）
log4j.appender.A1=org.apache.1og4j.ConsoleAppender
###2.3 配置日志信息的输出格式（Layout）
log4j.appender.A1.layout=org.apache.log4j.PatternLayout

log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>cn.itcast</groupId>
    <artifactId>itcast-crawler-first</artifactId>
    <version>1.0-SNAPSHOT</version>
        <dependencies>
            <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
            <dependency>
                <groupId>org.apache.httpcomponents</groupId>
                <artifactId>httpclient</artifactId>
                <version>4.5.2</version>
            </dependency>

            <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
            <dependency>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
                <version>1.7.25</version>
                <scope>test</scope>
            </dependency>


        </dependencies>

</project>

java

package cn.a.aa.test;


import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;


public class Demo{

    public static  void main(String[] args) throws Exception {
        //1.打开浏览器。创建一个httpclient对象
        //HttpClient(用CloseableHttpClient发送get/post请求)
//        HttpClient httpClient = new HttpClient();
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //2.输入网址,发起get请求，创建httpget对象
        HttpGet httpGet = new HttpGet("http://www.itcast.cn");

        //3.按回车，发起请求，返回响应,使用httpClient对象发送请求，相当于回车
        CloseableHttpResponse response = httpClient.execute(httpGet);
        //4.解析响应，获取数据
        //判断响应状态码是否是200，200就代表响应成功
        if (response.getStatusLine().getStatusCode()==200) {
            //得到响应体，比如请求网页的时候，返回的是网页的HTML代码
            HttpEntity httpEntity = response.getEntity();
            //对其进行解析，获得其里面的数据
            String content = EntityUtils.toString(httpEntity,"utf8");
            System.out.println(content);

        }

    }
}