首先从首页爬到所有教程链接并访问
// main.go
package main
import (
"fmt"
"log"
"strings"
"github.com/gocolly/colly/v2"
)
func main() {
// 创建数据库连接
db, err := InitDB()
if err != nil {
log.Fatal(err)
}
sqlDB, err := db.DB()
if err != nil {
log.Fatal(err)
}
defer sqlDB.Close()
// 创建一个新的爬虫实例
c := colly.NewCollector(
colly.AllowedDomains("www.runoob.com"),
)
// 定义一个列表来存储链接
var links []string
// 访问页面时的回调,获取 class 为 middle-column-home 中的 class 为 codelist-desktop 的链接
c.OnHTML(".middle-column-home .codelist-desktop a[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")
if link != "" && strings.HasPrefix(link, "/") {
links = append(links, e.Request.AbsoluteURL(link))
fmt.Println("Found link:", e.Request.AbsoluteURL(link))
}
})
// 错误处理
c.OnError(func(_ *colly.Response, err error) {
log.Println("Something went wrong:", err)
})
// 访问主页面
fmt.Println("Visiting main page...")
c.Visit("https://www.runoob.com")
// 等待爬虫完成
c.Wait()
// 创建一个新的爬虫实例来访问子页面
subCollector := c.Clone()
// 定义子页面爬取逻辑
defineChildPageLogic(subCollector, db)
// 访问提取到的所有链接
for _, link := range links {
fmt.Println("Visiting sub page:", link)
subCollector.Visit(link)
}
// 等待爬虫完成
subCollector.Wait()
}
访问每个链接时,爬取左侧菜单中的链接并访问
// child_page_handler.go
package main
import (
"fmt"
"log"
"strings"
"github.com/gocolly/colly/v2"
"gorm.io/gorm"
)
func defineChildPageLogic(c *colly.Collector, db *gorm.DB) {
handleContent := func(e *colly.HTMLElement, builder *strings.Builder) {
text := strings.TrimSpace(e.Text)
if text != "" {
builder.WriteString(text + "\n\n")
}
}
// 页面中的教程内容是放在.article-intro h1, .article-intro h2, .article-intro p这些标签中的
c.OnHTML(".article-intro h1, .article-intro h2, .article-intro p", func(e *colly.HTMLElement) {
if pageContent, ok := e.Request.Ctx.GetAny("pageContent").(*strings.Builder); ok {
handleContent(e, pageContent)
}
})
// 某个链接访问结束时存储
c.OnScraped(func(r *colly.Response) {
if pageContent, ok := r.Ctx.GetAny("pageContent").(*strings.Builder); ok && pageContent.Len() > 0 {
tableName := generateTableName(r.Request.URL.Path)
createTableIfNotExists(db, tableName)
title := r.Ctx.Get("title")
url := r.Request.URL.String()
content := pageContent.String()
saveToDatabase(db, tableName, title, url, content)
fmt.Println("Saved content for:", r.Request.URL.String())
}
})
// 子页面的爬虫实例中绑定 .left-column 的处理逻辑,网页中所有访问链接时放在.left-column中的a标签中
c.OnHTML(".left-column", func(e *colly.HTMLElement) {
e.ForEach("a", func(_ int, el *colly.HTMLElement) {
link := el.Attr("href")
if link != "" && strings.HasPrefix(link, "/") {
absoluteURL := "https://www.runoob.com" + link
// 创建新的爬虫实例来处理子子页面
childCollector := colly.NewCollector(
colly.AllowedDomains("www.runoob.com"),
)
defineSubSubPageLogic(childCollector, db)
ctx := colly.NewContext()
pageContent := new(strings.Builder)
ctx.Put("pageContent", pageContent)
ctx.Put("title", el.Text)
childCollector.Request("GET", absoluteURL, nil, ctx, nil)
}
})
})
c.OnError(func(_ *colly.Response, err error) {
log.Fatalf("请求错误: %v", err)
})
}
访问每个教程中左侧链接
// sub_sub_page_handler.go
package main
import (
"fmt"
"log"
"strings"
"github.com/gocolly/colly/v2"
"gorm.io/gorm"
)
func defineSubSubPageLogic(c *colly.Collector, db *gorm.DB) {
handleContent := func(e *colly.HTMLElement, builder *strings.Builder) {
text := strings.TrimSpace(e.Text)
if text != "" {
builder.WriteString(text + "\n\n")
}
}
c.OnHTML(".article-intro h1, .article-intro h2, .article-intro p", func(e *colly.HTMLElement) {
if pageContent, ok := e.Request.Ctx.GetAny("pageContent").(*strings.Builder); ok {
handleContent(e, pageContent)
}
})
c.OnScraped(func(r *colly.Response) {
if pageContent, ok := r.Ctx.GetAny("pageContent").(*strings.Builder); ok && pageContent.Len() > 0 {
tableName := generateTableName(r.Request.URL.Path)
createTableIfNotExists(db, tableName)
title := r.Ctx.Get("title")
url := r.Request.URL.String()
content := pageContent.String()
saveToDatabase(db, tableName, title, url, content)
fmt.Println("Saved content for:", r.Request.URL.String())
}
})
c.OnError(func(_ *colly.Response, err error) {
log.Fatalf("请求错误: %v", err)
})
}
连接数据库
// db.go
package main
import (
"fmt"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"log"
)
func InitDB() (*gorm.DB, error) {
dsn := "root:111111@tcp(192.168.0.10:3306)/rolly_reptiles?charset=utf8mb4&parseTime=True&loc=Local"
db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
if err != nil {
return nil, err
}
return db, nil
}
func CloseDB(db *gorm.DB) {
sqlDB, err := db.DB()
if err != nil {
log.Fatalf("无法关闭数据库连接: %v", err)
}
sqlDB.Close()
}
func createTableIfNotExists(db *gorm.DB, tableName string) {
createTableQuery := fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS %s (
id INT AUTO_INCREMENT PRIMARY KEY,
title VARCHAR(255) NOT NULL,
url VARCHAR(255) NOT NULL,
content TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
`, tableName)
if err := db.Exec(createTableQuery).Error; err != nil {
log.Fatalf("无法创建表: %v", err)
}
}
保存
// utils.go
package main
import (
"fmt"
"gorm.io/gorm"
"log"
"regexp"
"strings"
)
func getSubPageIdentifier(path string) string {
parts := strings.Split(path, "/")
if len(parts) > 1 {
return "/" + parts[1]
}
return ""
}
func generateTableName(path string) string {
parts := strings.Split(path, "/")
if len(parts) > 1 {
// 使用正则表达式去除非法字符
re := regexp.MustCompile(`[^a-zA-Z0-9_]`)
tableName := re.ReplaceAllString(parts[1], "_")
return fmt.Sprintf("articles_%s", tableName)
}
return "articles_default"
}
func saveToDatabase(db *gorm.DB, tableName, title, url, content string) {
insertQuery := fmt.Sprintf("INSERT INTO %s (title, url, content) VALUES (?, ?, ?)", tableName)
if err := db.Exec(insertQuery, title, url, content).Error; err != nil {
log.Fatalf("无法插入数据到数据库: %v", err)
}
fmt.Println("内容已保存到数据库")
}
go mod
module colly_sutdy
go 1.22.3
require github.com/gocolly/colly/v2 v2.1.0
require (
github.com/PuerkitoBio/goquery v1.5.1 // indirect
github.com/andybalholm/cascadia v1.2.0 // indirect
github.com/antchfx/htmlquery v1.2.3 // indirect
github.com/antchfx/xmlquery v1.2.4 // indirect
github.com/antchfx/xpath v1.1.8 // indirect
github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.4.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/temoto/robotstxt v1.1.1 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/protobuf v1.24.0 // indirect
)
require (
gorm.io/driver/mysql v1.5.1
gorm.io/gorm v1.25.1
)