一个并行的go爬虫

1.使用get函数获取当前页面内容以string返回

func Get(url string)(result string,err error){
	resp, err1 := http.Get(url)
	if err != nil {
		err = err1
		return
	}
	defer  resp.Body.Close()

	buf := make([]byte,4 * 1024)
	for true{
		n,err := resp.Body.Read(buf)
		if err != nil{
			if err == io.EOF{
				fmt.Println("文件读取完毕")
				break
			}else {
				fmt.Println("resp.Body.Read failed err = ",err)
				break
			}
		}
		result += string(buf[:n])
	}
	return result,err
}

2.定义函数来循环不同页面，将每页的文件保存到文件当中。

func SpiderPage(i int,page chan int) {
	url := "https://github.com/search?q=go&type=Repositories&p=1"+strconv.Itoa((i - 1) * 50)
	fmt.Println("正在爬取第",i,"个网页")
	result,err := Get(url)
	if err != nil{
		fmt.Println("http.Get failed err = ",err)
		return
	}
	fileName := "page"+strconv.Itoa(i)+".html"
	f, err1 := os.Create(fileName)
	if err != nil{
		log.Fatalln("Create failed err = ",err1)
		return
	}
	f.WriteString(result)
	f.Close()
	page <- 1
}

3.使用go关键字，让每一页的文件读取都另开一个进程

func Run(start,end int){
	fmt.Println("正在爬取第",start,"页到第",end,"页")
	page := make(chan int)
	for i := start;i <= end;i++{
		go SpiderPage(i,page)
	}
	for i := start;i <= end;i++{
		fmt.Printf("第%d个页面爬取完成\n",<-page)
	}
}

4.主函数直接运行文件

func main() {
	var start,end int
	fmt.Println("请输入起始页数字")
	fmt.Scan(&start)
	fmt.Println("请输入结束页数字")
	fmt.Scan(&end)
	Run(start,end)
}