go爬虫练习，爬取菜鸟教程

首先从首页爬到所有教程链接并访问

// main.go
package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/gocolly/colly/v2"
)

func main() {
	// 创建数据库连接
	db, err := InitDB()
	if err != nil {
		log.Fatal(err)
	}
	sqlDB, err := db.DB()
	if err != nil {
		log.Fatal(err)
	}
	defer sqlDB.Close()

	// 创建一个新的爬虫实例
	c := colly.NewCollector(
		colly.AllowedDomains("www.runoob.com"),
	)

	// 定义一个列表来存储链接
	var links []string

	// 访问页面时的回调，获取 class 为 middle-column-home 中的 class 为 codelist-desktop 的链接
	c.OnHTML(".middle-column-home .codelist-desktop a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		if link != "" && strings.HasPrefix(link, "/") {
			links = append(links, e.Request.AbsoluteURL(link))
			fmt.Println("Found link:", e.Request.AbsoluteURL(link))
		}
	})

	// 错误处理
	c.OnError(func(_ *colly.Response, err error) {
		log.Println("Something went wrong:", err)
	})

	// 访问主页面
	fmt.Println("Visiting main page...")
	c.Visit("https://www.runoob.com")

	// 等待爬虫完成
	c.Wait()

	// 创建一个新的爬虫实例来访问子页面
	subCollector := c.Clone()

	// 定义子页面爬取逻辑
	defineChildPageLogic(subCollector, db)

	// 访问提取到的所有链接
	for _, link := range links {
		fmt.Println("Visiting sub page:", link)
		subCollector.Visit(link)
	}

	// 等待爬虫完成
	subCollector.Wait()
}

访问每个链接时，爬取左侧菜单中的链接并访问

// child_page_handler.go
package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/gocolly/colly/v2"
	"gorm.io/gorm"
)

func defineChildPageLogic(c *colly.Collector, db *gorm.DB) {
	handleContent := func(e *colly.HTMLElement, builder *strings.Builder) {
		text := strings.TrimSpace(e.Text)
		if text != "" {
			builder.WriteString(text + "\n\n")
		}
	}
	// 页面中的教程内容是放在.article-intro h1, .article-intro h2, .article-intro p这些标签中的
	c.OnHTML(".article-intro h1, .article-intro h2, .article-intro p", func(e *colly.HTMLElement) {
		if pageContent, ok := e.Request.Ctx.GetAny("pageContent").(*strings.Builder); ok {
			handleContent(e, pageContent)
		}
	})
	// 某个链接访问结束时存储
	c.OnScraped(func(r *colly.Response) {
		if pageContent, ok := r.Ctx.GetAny("pageContent").(*strings.Builder); ok && pageContent.Len() > 0 {
			tableName := generateTableName(r.Request.URL.Path)
			createTableIfNotExists(db, tableName)
			title := r.Ctx.Get("title")
			url := r.Request.URL.String()
			content := pageContent.String()
			saveToDatabase(db, tableName, title, url, content)
			fmt.Println("Saved content for:", r.Request.URL.String())
		}
	})

	// 子页面的爬虫实例中绑定 .left-column 的处理逻辑，网页中所有访问链接时放在.left-column中的a标签中
	c.OnHTML(".left-column", func(e *colly.HTMLElement) {
		e.ForEach("a", func(_ int, el *colly.HTMLElement) {
			link := el.Attr("href")
			if link != "" && strings.HasPrefix(link, "/") {
				absoluteURL := "https://www.runoob.com" + link

				// 创建新的爬虫实例来处理子子页面
				childCollector := colly.NewCollector(
					colly.AllowedDomains("www.runoob.com"),
				)
				defineSubSubPageLogic(childCollector, db)

				ctx := colly.NewContext()
				pageContent := new(strings.Builder)
				ctx.Put("pageContent", pageContent)
				ctx.Put("title", el.Text)

				childCollector.Request("GET", absoluteURL, nil, ctx, nil)
			}
		})
	})

	c.OnError(func(_ *colly.Response, err error) {
		log.Fatalf("请求错误: %v", err)
	})
}

 访问每个教程中左侧链接

// sub_sub_page_handler.go
package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/gocolly/colly/v2"
	"gorm.io/gorm"
)

func defineSubSubPageLogic(c *colly.Collector, db *gorm.DB) {
	handleContent := func(e *colly.HTMLElement, builder *strings.Builder) {
		text := strings.TrimSpace(e.Text)
		if text != "" {
			builder.WriteString(text + "\n\n")
		}
	}

	c.OnHTML(".article-intro h1, .article-intro h2, .article-intro p", func(e *colly.HTMLElement) {
		if pageContent, ok := e.Request.Ctx.GetAny("pageContent").(*strings.Builder); ok {
			handleContent(e, pageContent)
		}
	})

	c.OnScraped(func(r *colly.Response) {
		if pageContent, ok := r.Ctx.GetAny("pageContent").(*strings.Builder); ok && pageContent.Len() > 0 {
			tableName := generateTableName(r.Request.URL.Path)
			createTableIfNotExists(db, tableName)
			title := r.Ctx.Get("title")
			url := r.Request.URL.String()
			content := pageContent.String()
			saveToDatabase(db, tableName, title, url, content)
			fmt.Println("Saved content for:", r.Request.URL.String())
		}
	})

	c.OnError(func(_ *colly.Response, err error) {
		log.Fatalf("请求错误: %v", err)
	})
}

 连接数据库

// db.go
package main

import (
	"fmt"
	"gorm.io/driver/mysql"
	"gorm.io/gorm"
	"log"
)

func InitDB() (*gorm.DB, error) {
	dsn := "root:111111@tcp(192.168.0.10:3306)/rolly_reptiles?charset=utf8mb4&parseTime=True&loc=Local"
	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
	if err != nil {
		return nil, err
	}
	return db, nil
}

func CloseDB(db *gorm.DB) {
	sqlDB, err := db.DB()
	if err != nil {
		log.Fatalf("无法关闭数据库连接: %v", err)
	}
	sqlDB.Close()
}

func createTableIfNotExists(db *gorm.DB, tableName string) {
	createTableQuery := fmt.Sprintf(`
	CREATE TABLE IF NOT EXISTS %s (
		id INT AUTO_INCREMENT PRIMARY KEY,
		title VARCHAR(255) NOT NULL,
		url VARCHAR(255) NOT NULL,
		content TEXT NOT NULL,
		created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
	) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
	`, tableName)

	if err := db.Exec(createTableQuery).Error; err != nil {
		log.Fatalf("无法创建表: %v", err)
	}
}

保存 

// utils.go
package main

import (
	"fmt"
	"gorm.io/gorm"
	"log"
	"regexp"
	"strings"
)

func getSubPageIdentifier(path string) string {
	parts := strings.Split(path, "/")
	if len(parts) > 1 {
		return "/" + parts[1]
	}
	return ""
}

func generateTableName(path string) string {
	parts := strings.Split(path, "/")
	if len(parts) > 1 {
		// 使用正则表达式去除非法字符
		re := regexp.MustCompile(`[^a-zA-Z0-9_]`)
		tableName := re.ReplaceAllString(parts[1], "_")
		return fmt.Sprintf("articles_%s", tableName)
	}
	return "articles_default"
}

func saveToDatabase(db *gorm.DB, tableName, title, url, content string) {
	insertQuery := fmt.Sprintf("INSERT INTO %s (title, url, content) VALUES (?, ?, ?)", tableName)
	if err := db.Exec(insertQuery, title, url, content).Error; err != nil {
		log.Fatalf("无法插入数据到数据库: %v", err)
	}
	fmt.Println("内容已保存到数据库")
}

 go mod

module colly_sutdy

go 1.22.3

require github.com/gocolly/colly/v2 v2.1.0

require (
	github.com/PuerkitoBio/goquery v1.5.1 // indirect
	github.com/andybalholm/cascadia v1.2.0 // indirect
	github.com/antchfx/htmlquery v1.2.3 // indirect
	github.com/antchfx/xmlquery v1.2.4 // indirect
	github.com/antchfx/xpath v1.1.8 // indirect
	github.com/go-sql-driver/mysql v1.7.0 // indirect
	github.com/gobwas/glob v0.2.3 // indirect
	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
	github.com/golang/protobuf v1.4.2 // indirect
	github.com/jinzhu/inflection v1.0.0 // indirect
	github.com/jinzhu/now v1.1.5 // indirect
	github.com/kennygrant/sanitize v1.2.4 // indirect
	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
	github.com/stretchr/testify v1.8.4 // indirect
	github.com/temoto/robotstxt v1.1.1 // indirect
	golang.org/x/net v0.21.0 // indirect
	golang.org/x/text v0.14.0 // indirect
	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
	google.golang.org/appengine v1.6.6 // indirect
	google.golang.org/protobuf v1.24.0 // indirect
)

require (
	gorm.io/driver/mysql v1.5.1
	gorm.io/gorm v1.25.1
)