简单爬取百度贴吧例子
package main
import (
"fmt"
"strconv"
"net/http"
"io"
"os"
)
func httpGet(url string) (result string, err error) {
resp, err1 := http.Get(url)
if err != nil {
err = err1
return
}
defer resp.Body.Close()
buf := make([]byte, 4096)
for {
n, err2 := resp.Body.Read(buf)
if n == 0 {
fmt.Println("读取网页完成")
break
}
if err2 != nil && err2 != io.EOF {
err = err2
return
}
result += string(buf[:n])
}
return
}
func spiderFun(url string, i int, page chan <- int) {
result, err := httpGet(url)
if err != nil {
fmt.Println("httpGet err:", err)
return
}
// fmt.Println("result = ", result)
f, err := os.Create("第"+ strconv.Itoa(i) + "页" + ".html")
if err != nil {
fmt.Println("os.Create err:", err)
return
}
f.WriteString(result)
f.Close()
page <- i
}
func working(start, end int) {
page := make(chan int)
for i := start; i <= end; i++ {
url := "https://tieba.baidu.com/f?kw=lol&ie=utf-8&pn=" + strconv.Itoa((i-1)*50)
// fmt.Println(url)
go spiderFun(url, i, page)
}
// 与主go程完成同步
for i := start; i <= end; i++ {
fmt.Printf("爬取第%d个页面完成", <- page)
}
}
func main() {
var start, end int
fmt.Print("请输入爬取的起始页(>=1)")
fmt.Scan(&start)
fmt.Print("请输入爬取的起始页(>=start)")
fmt.Scan(&end)
working(start, end)
}
正则
package main
import(
"regexp"
"fmt"
)
func main() {
str := "ssabnabw 2.3 344 5.4"
str2 := `
<div>
hello world
</div>
`
reg, _ := regexp.Compile(`\d+\.\d+`)
ret := reg.FindAllStringSubmatch(str, -1) //2.3 5.4
fmt.Println(ret)
// 将(?s:(.*?))元组放置在某一特征字符串中就可以提取这一字符串的内容
reg, _ = regexp.Compile(`<div>(?s:(.*?))</div>`)
ret = reg.FindAllStringSubmatch(str2, -1)
fmt.Println(ret[])
/*
<div>
hello world
</div>
*/
}