goquery的使用

qq_37694877

已于 2022-04-12 17:01:03 修改

阅读量393

点赞数 1

文章标签： go

于 2022-04-11 22:56:28 首次发布

本文链接：https://blog.csdn.net/qq_37694877/article/details/124111197

版权

这段代码展示了如何使用GoQuery库从一个网页中提取多个链接。它首先发送HTTP请求获取页面，然后解析HTML，针对特定CSS选择器找到链接，并过滤掉包含"index"的链接。主要涉及网络爬虫和Go语言编程。

摘要由CSDN通过智能技术生成

首先需要在github中找到对应的包

根据标签的内容进行处理

//fun1.go
package fun1

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"log"
	"net/http"
	"strings"
)

// Request the HTML page.

var urls []string
func UrlFunc1() []string{
	res, err := http.Get("一个网址")
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
	}

	// Load the HTML document
	doc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	doc.Find("#rm_aq > h2 > a").Each(func(i int, selection *goquery.Selection) {
		fmt.Printf("6:")
		description, _ := selection.Attr("href")
		description=strings.Replace(description,"\t","",-1)
		description=strings.Replace(description,"\n","",-1)
		description=strings.Replace(description," ","",-1)
		fmt.Println( description)
		if strings.Contains(description,"index")==false{
			urls =append(urls,description)
		}

	})

	doc.Find("#rm_aq > ul:nth-child(2)>li>a").Each(func(i int, selection *goquery.Selection) {

		fmt.Printf("7:")
		description, _ := selection.Attr("href")
		description=strings.Replace(description,"\t","",-1)
		description=strings.Replace(description,"\n","",-1)
		description=strings.Replace(description," ","",-1)
		fmt.Println( description)
		if strings.Contains(description,"index")==false{
			urls =append(urls,description)
		}
	})

	doc.Find("#rm_aq > div.fenjinnew > ul > li>a").Each(func(i int, selection *goquery.Selection) {

		fmt.Printf("8:")
		description, _ := selection.Attr("href")
		description=strings.Replace(description,"\t","",-1)
		description=strings.Replace(description,"\n","",-1)
		description=strings.Replace(description," ","",-1)
		fmt.Println( description)
		if strings.Contains(description,"index")==false{
			urls =append(urls,description)
		}
	})
	doc.Find("#rm_aq > div.a6zw > h2 > a").Each(func(i int, selection *goquery.Selection) {

		fmt.Printf("10:")
		description, _ := selection.Attr("href")
		description=strings.Replace(description,"\t","",-1)
		description=strings.Replace(description,"\n","",-1)
		description=strings.Replace(description," ","",-1)
		fmt.Println( description)
		if strings.Contains(description,"index")==false{
			urls =append(urls,description)
		}
	})

	doc.Find("#rm_aq > ul:nth-child(7)> li>a").Each(func(i int, selection *goquery.Selection) {

		fmt.Printf("9:")
		description, _ := selection.Attr("href")
		description=strings.Replace(description,"\t","",-1)
		description=strings.Replace(description,"\n","",-1)
		description=strings.Replace(description," ","",-1)
		fmt.Println( description)
		if strings.Contains(description,"index")==false{
			urls =append(urls,description)
		}
	})
	return urls
}