Golang——爬取网页数据时403

情景：浏览器通过url能正常访问，但是使用http.get请求报403

修改前的实现

func Fetcher(url string) ([]byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		fmt.Println("Error status code:", resp.StatusCode)
		return nil, fmt.Errorf("wrong http status code: %d", resp.StatusCode)
	}

	// 编码转换
	//htmlEncoding:=determineEncoding(resp.Body)
	//reader := transform.NewReader(resp.Body, htmlEncoding.NewDecoder())
	//body, err := ioutil.ReadAll(reader)

	return ioutil.ReadAll(resp.Body)
}

改动后的实现

func Fetcher(url string) ([]byte, error) {
	client := http.Client{}
	request, err := http.NewRequest("GET", url, nil)
	if err!=nil {
		fmt.Printf("wrong http request: %s", err.Error())
		return nil, fmt.Errorf("wrong http request: %s", err.Error())
	}
	request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36")
	resp, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		fmt.Println("Error status code:", resp.StatusCode)
		return nil, fmt.Errorf("wrong http status code: %d", resp.StatusCode)
	}

	// 编码转换
	//htmlEncoding:=determineEncoding(resp.Body)
	//reader := transform.NewReader(resp.Body, htmlEncoding.NewDecoder())
	//body, err := ioutil.ReadAll(reader)

	return ioutil.ReadAll(resp.Body)
}

关键点

设置http header中的User-Agent

request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36")

参考：https://blog.csdn.net/qq_36183935/article/details/80499183