24-golang简单小爬虫

 

 

来写一个简单的小爬虫

只是小试牛刀

只是学点皮毛

 

 

func main() {

   var page int

   fmt.Println("请输入想要爬取的页面")
   fmt.Scan(&page)

   result := HandlePage(page)

   name := strconv.Itoa(page)
   file, err := os.Create(name + ".html")
   if err != nil {
      fmt.Println("err = ", err)
      return
   }

   defer file.Close()

   file.WriteString(result)

}

func HandlePage(page int) (result string) {

   url := "https://tieba.baidu.com/f?kw=%E6%B5%B7%E8%B4%BC%E7%8E%8B&ie=utf-8&pn=" + strconv.Itoa(page*50)

   resp, err := http.Get(url)

   if err != nil {
      fmt.Println("err = ", err)
      return
   }

   defer resp.Body.Close()

   bytes := make([]byte, 1024)

   for {
      num, err := resp.Body.Read(bytes)
      if err == io.EOF {
         break
      }

      result += string(bytes[:num])

      if err != nil {
         fmt.Println("err = ", err)
         return
      }
   }

   return
}