http.Client默认的User-Agent是“Go-http-client/1.1”,在做页面抓取的时候容易被服务端拒绝,需要在请求前重新设置一个模拟的User-Agent。具体代码如下:
package main
import (
"fmt"
"io/ioutil"
"net/http"
"time"
)
func main() {
client := &http.Client{
Timeout: 2 * time.Second,
}
req, _ := http.NewRequest("GET", "http://127.0.0.1:8080", nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
fmt.Printf("%s\n", err.Error())
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Printf("%s\n", err.Error())
return
}
fmt.Printf("%s\n", string(body))
}
服务端验证代码:
package main
import (
"fmt"
"io"
"log"
"net/http"
)
func main() {
// Hello world, the web server
helloHandler := func(w http.ResponseWriter, r *http.Request) {
fmt.Printf("%v\n", r.UserAgent())
io.WriteString(w, "Hello, world!\n")
}
http.HandleFunc("/", helloHandler)
log.Fatal(http.ListenAndServe(":8080", nil))
}
请求后,服务端输出:
Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36