package main
import (
"fmt"
"net/http"
"os"
"strconv"
)
//爬取网页内容
func HttpGet(url string)(result string ,err error){
resp , err1 :=http.Get(url)
if err1 !=nil{
err = err1
return
}
defer resp.Body.Close()
//读取网页body内容
buf := make([]byte, 1024*4 )
for{
n, err := resp.Body.Read(buf)
//读取结束,或者出问题
if n == 0 {
fmt.Println("resp.Body.Read err = ",err)
break
}
result += string(buf[:n])
}
return
}
func DoWork(start,end int){
fmt.Printf("正在爬取 %d 到 %d 的页面",start,end)
//每一页+50 https://tieba.baidu.com/f?kw=%E7%BB%9D%E5%9C%B0%E6%B1%82%E7%94%9F&ie=utf-8&pn=50
for i:=start;i<=end;i++{
url := "https://tieba.baidu.com/f?kw=%E7%BB%9D%E5%9C%B0%E6%B1%82%E7%94%9F&ie=utf-8&pn="+strconv.Itoa((i-1)*50)
fmt.Println("url = ",url)
result, err := HttpGet(url)
if err !&#
goLang爬取html
最新推荐文章于 2024-07-06 14:57:35 发布
![](https://img-home.csdnimg.cn/images/20240711042549.png)