一、背景
准备完善之前的小说爬虫(19年年底编写的项目),运行后,即用go发起http的GET请求,返回400错误码。
二、出现的问题
请求代码
func bookScan() {
url := "https://m.999xs.com/search.php?keyword=完美世界"
client := http.Client{}
request, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("http.NewRequest error: %s", err)
}
request.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36")
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
request.Header.Set("Connection", "keep-alive")
resp, err := client.Do(request)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
log.Fatalf("status code error: %d %s", resp.StatusCode, resp.Status)
}
}
返回消息:
2021/07/09 00:45:27 status code error: 400 400 Bad Request
exit status 1
用postman及浏览器测试皆没有问题。
三、解决方案(modify注释的地方)
func bookScan() {
url := "https://m.999xs.com/search.php?keyword=完美世界"
client := http.Client{}
request, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalf("http.NewRequest error: %s", err)
}
// ----- modify start --------
q := request.URL.Query()
q.Add("keyword", "完美世界")
request.URL.RawQuery = q.Encode()
// ----- modify end --------
request.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36")
request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
request.Header.Set("Connection", "keep-alive")
resp, err := client.Do(request)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
log.Fatalf("status code error: %d %s", resp.StatusCode, resp.Status)
}
}