using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HtmlAgilityPack;
using System.Net;
using System.IO;
namespace htmlagilitypackDemo
{
class Program
{
private const string CategoryListXPath = "./div[2]/div"; //关键点,不同网站分析不同的路径
private const string CategoryNameXPath = "./table[1]/tr[1]/td[2]/h3[1]"; //关键点,不同网站分析不同的路径
private const string ChooseXPath = "./a[1]";
static void Main(string[] args)
{
Demo_1();
Console.Read();
}
static void Demo_1()
{
Uri url = new Uri("http://news.sogou.com/news?query=%BA%A3%B6%FB&p=42230305&dp=1&_ast=1353663787&_asf=news.sogou.com&time=0&w=03009900&sort=1&mode=1&manual=");
Uri uriCategory = null;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
WebResponse response = request.GetResponse();
Stream stream = response.GetResponseStream();
StreamReader read = new StreamReader(stream, Encoding.GetEncoding("gb2312"));
string str = read.ReadToEnd();
HtmlDocument html = new HtmlDocument();
html.LoadHtml(str);
//HtmlNode rootNode = html.DocumentNode;
HtmlNode rootNode = html.GetElementbyId("main");
HtmlNodeCollection categoryNodeList = rootNode.SelectNodes(CategoryListXPath);
HtmlNode temp = null;
var list = new List<Category>();
foreach (HtmlNode categoryNode in categoryNodeList)
{
temp = HtmlNode.CreateNode(categoryNode.OuterHtml);
HtmlNode singleNode = temp.SelectSingleNode(CategoryNameXPath);
if (singleNode == null)
continue;
HtmlNodeCollection singleList = temp.SelectNodes(CategoryNameXPath);
foreach (HtmlNode node in singleList)
{
HtmlNode createNode = HtmlNode.CreateNode(node.OuterHtml);
HtmlNode reNode = createNode.SelectSingleNode(ChooseXPath);
if (reNode == null)
continue;
Category category = new Category();
category.Subject = reNode.InnerText;
Uri.TryCreate(url, reNode.Attributes["href"].Value, out uriCategory);
category.IndexUrl = uriCategory.ToString();
list.Add(category);
Console.WriteLine("name:{0}",category.Subject);
Console.WriteLine("url:{0}",category.IndexUrl);
//break;
}
}
}
}
public class Category
{
public string Subject { get; set; }
public string IndexUrl { get; set; }
}
}
htmlagilitypackDemo
最新推荐文章于 2018-12-24 11:40:45 发布