php怎么爬小说,记一次GO爬取小说

示意图

package main

import (

"fmt"

"github.com/gocolly/colly"

"os"

)

func main() {

url:="https://www.biqiuge.com/book/38767/"

fileTitle := make([]string,0)

filecontent := make([]string,0)

c := colly.NewCollector()

c.AllowURLRevisit = true

c.DetectCharset = true

contentCollector := c.Clone()

beginRevist := false

selects := "div[class=listmain]"

c.OnHTML(selects, func(element *colly.HTMLElement) {

element.ForEach("dd>a", func(_ int, eleHref *colly.HTMLElement) {

tmpurl := eleHref.Attr("href")

//if strings.Index(eleHref.Text,"第一章")!= -1{

fileTitle = append(fileTitle,eleHref.Text)

beginRevist = true

//}

if beginRevist{

chapteurl := eleHref.Request.AbsoluteURL(tmpurl)

contentCollector.Visit(chapteurl)

}

})

})

c.OnRequest(func(request *colly.Request) {

fmt.Println("visiting",request.URL.String())

})

contselect := "div[class=showtxt]"

contentCollector.OnHTML(contselect, func(elementcont *colly.HTMLElement) {

//fmt.Printf("%s\n",elementcont.Text)

filecontent = append(filecontent,elementcont.Text)

})

c.Visit(url)

/*

fmt.Println(len(filecontent))

fmt.Println(len(fileTitle))

*/

filenum := len(fileTitle)

for i:=0;i

//fmt.Println(reflect.TypeOf(fileTitle[i]))

strpath ,_:= os.Getwd()

path := strpath + "/"+fileTitle[i]+".txt"

f, err := os.Create(path)

if err != nil {

fmt.Println(err)

}

f.WriteString(filecontent[i])

//fmt.Printf("正在下载%s",fileTitle[i])

defer f.Close()

}

}

有疑问加站长微信联系(非本文作者)