C#AngelSharp爬取豆瓣组及微博组所有帖子

以前没写过爬虫，百度了豆瓣、微博爬取，得到的答案基本都是几年前事了，没多少可用的信息，就自己研究了下，做下整理，以下仅放出关键解析代码，有需要软件源码的回复下，足够多就放出链接。
注意：在爬取时设置一个延时时间，使用Task.Delay()设置个几秒，不设置爬取不了几个帖子就会被封杀IP，需要更换代理才能继续

//通过组url获取所有帖子

var config = Configuration.Default.WithDefaultLoader();
//address= 豆瓣组URL，使用async await异步实现网页拉取
var document = await 
//address = https://www.douban.com/group/703309/
BrowsingContext.New(config).OpenAsync(address);
//获取所有IHtmlElement元素
var all = document.QuerySelectorAll<IHtmlElement>("a");
 foreach (var el in all) {
          string href = el.GetAttribute("href");
          if (href != null && href.Contains("group/topic") && el.ClassName == "" && el.Title != null) {
   // 这里得到帖子URL
                 result.Enqueue(href);
           }
  }

//获取帖子信息

var config = Configuration.Default.WithDefaultLoader();
//address=帖子地址，通过组获取
var document = await BrowsingContext.New(config).OpenAsync(address);
var cells = document.QuerySelectorAll("script ");
//分析html元素并解析Json，拿我们需要的内容
foreach (IHtmlScriptElement cell in cells)
{
    if (cell.Type != null && cell.Type.Equals("application/ld+json"))
    {
        string json = cell.Text;
        json = json.Replace("\t", "");
        json = json.Replace("\n", "");
        var des = JObject.Parse(json);
        Dictionary<string, JToken> map = new Dictionary<string, JToken>();
        foreach (JToken child in des.Children())
        {
            var property1 = child as JProperty;
            map.Add(property1.Name, property1.Value);
        }
//帖子标题
        data.Title = map["name"].ToString();
//帖子内容
        data.Contents = map["text"].ToString();
//帖子评论数量
        data.CommentsCount = int.Parse(map["commentCount"].ToString());
//发帖时间
        data.CreateDate = map["dateCreated"].ToString();
//其它map["..."]
    }
}

//获取作者发布帖子内容

var contentList = document.All.Where(m => m.ClassName == "topic-content clearfix");
if (contentList == null) return;
var cDocumet = contentList.First();
var allP = cDocumet.QuerySelectorAll<IHtmlElement>("p");
var allImg = cDocumet.QuerySelectorAll("img");
data.Contents = "";
foreach (var p in allP)
{
    data.Contents += p.TextContent + "\n";
    var aList = p.QuerySelectorAll("a");    //获取innerText内超链接
    if (aList != null)
    {
        foreach (IHtmlElement a in aList)
        {
//所有超链接
            List<string> tempList = getLink(a);
        }
    }
}
foreach (var img in allImg)
{
    string alt = img.GetAttribute("alt");
    string src = img.GetAttribute("src");
    if (alt == null)
    {
        //帖子图片
        data.ResoursAddress.Add(src);
    }
    else if(img.ClassName== "pil") {
        //作者头像及名字
        data.AuthorFace = src;
        data.AuthorName = alt;
    }
}

//解析评论回复

var compsList = document.All.Where(m => m.ClassName == "clearfix comment-item reply-item ");
data.CommentList = new List<CommentContent>();
foreach (var comps in compsList)
{
    CommentContent comment = new CommentContent();
    var spanList = comps.QuerySelectorAll<IHtmlElement>("span");
    var imgList = comps.QuerySelectorAll<IHtmlElement>("img");
    foreach (var img in imgList) {
        string alt = img.GetAttribute("alt");
        string src = img.GetAttribute("src");
        if (img.ClassName == "pil" && alt != null && src != null) {
            comment.Face = src; //评论人头像地址
            comment.User = alt; //评论人名字
        }
    }
    foreach (var span in spanList)
    {
        if (span.ClassName == "pubtime")
        {
            comment.Time = span.InnerHtml;
            break;
        }
    }
    var pList = comps.QuerySelectorAll<IHtmlElement>("p");  
    comment.Content = "";
    foreach (var p in pList)
    {
        comment.Content += p.TextContent + "\r\n";
        List<string> tempList = getLink(p);
        if (comment.Link == null) comment.Link = new List<string>();
        comment.Link.AddRange(tempList);
    }
    data.CommentList.Add(comment);
}          

明天放出微博解析代码，相比豆瓣解析复杂一些