C#使用HtmlAgilityPack爬虫实例

最新推荐文章于 2024-08-13 08:29:05 发布

cxhgg

最新推荐文章于 2024-08-13 08:29:05 发布

阅读量4.9k

点赞数 2

分类专栏： C#爬虫文章标签：爬虫 C# HtmlAgilityPack Request html

本文链接：https://blog.csdn.net/cxhgg/article/details/60959815

版权

C#爬虫专栏收录该内容

1 篇文章 0 订阅

订阅专栏

使用HtmlAgilityPack类库解析html非常方便，网上的资料有很多，可以自行搜索了解

下面上一个非常简单的小例子

要爬取的信息如下：

首先要引用HtmlAgilityPack.dll文件

上代码：

 internal void Run()
        {
            string url = "爬取网站URL";
            string res = HttpTool.Excute(url); //发送请求得到页面
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(res); //加载html
            HtmlNode documentNode = doc.DocumentNode;
            string hname = documentNode.SelectSingleNode("//div[@class='main-title']/h1").InnerText.Trim(); //定位到节点获取文本值
            string address = documentNode.SelectSingleNode("//div[@class='location']/span").InnerText.Trim();
        }

HttpTool类

using System;
using System.IO;
using System.Text;
using System.Net;


namespace Spider
{
    public class HttpTool : IDisposable
    {
        public static string Excute(string url, string data = null)
        {

            for (int i = 0; i < 3; i++)  //尝试3次连接
            {
                try
                {
                    using (HttpTool tool = new HttpTool(url, data))
                    {
                        return tool.Result;
                    }
                }
                catch (Exception ex)
                {
                    //异常处理
                }
            }
            return null;
        }

        public string Url { get; set; }

        public HttpWebRequest Request { get; private set; }

        public HttpWebResponse Response { get; private set; }

        public string Result { get; private set; }

        public HttpTool(string url, string postData = null)
        {
            Url = url;
            InitRequest();
            if (string.IsNullOrEmpty(postData))
            {
                InitResponse();
            }
            else
            {
                InitResponse(postData);
            }
            var stream = Response.GetResponseStream();
            var sr = new StreamReader(Response.GetResponseStream());
            Result = sr.ReadToEnd();
            stream.Dispose();
            sr.Close();
            sr.Dispose();
        }

        /// <summary>
        /// 初始化请求 请求头信息可以按自己需求增加
        /// </summary>
        public virtual void InitRequest()
        {

            Request = (HttpWebRequest)WebRequest.Create(Url);

            //Request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; QQWubi 133; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; CIBA; InfoPath.2)";
            //Request.KeepAlive = false;
            Request.Accept = "Accept	text/html, application/xhtml+xml, */*";
            Request.UserAgent = "User-Agent	Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)";
            Request.Timeout = 6000;
            Request.ReadWriteTimeout = 6000;
        }



        /// <summary>
        /// 初始化响应报文，GET
        /// </summary>
        public void InitResponse()
        {
            Request.Method = "GET";
            Response = (HttpWebResponse)Request.GetResponse();
        }

        /// <summary>
        /// 初始化响应报文，POST
        /// </summary>
        /// <param name="postData">POST参数</param>
        /// <param name="encoding">编码方式</param>
        public void InitResponse(string postData, Encoding encoding)
        {
            Request.Method = "POST";
            byte[] bs = encoding.GetBytes(postData);
            Request.ContentLength = bs.Length;
            var reqStream = Request.GetRequestStream();
            reqStream.Write(bs, 0, bs.Length);
            reqStream.Close();
            reqStream.Dispose();
            Response = (HttpWebResponse)Request.GetResponse();
        }

        /// <summary>
        /// 初始化响应报文，POST，UTF-8序列化
        /// </summary>
        /// <param name="postData">POST参数</param>
        public void InitResponse(string postData)
        {
            InitResponse(postData, Encoding.UTF8);
        }

    }

}