Golang原生实现Akamai/JA3指纹修改,并支持Proxy代理

起因

抓取某个HTTPS网站的时候
开启charles代理能够抓取成功,关闭被风控
通过检测,怀疑可能是tls的时候有区别

尝试

golang的http中,Transport.TLSClientConfig是可以自定义设置的
但起初通过随意设置并不能绕过风控

困难

1. 使用golang的http客户端,修改DialTLSContext函数的方式是可以实绕过风控,但使用proxy的时候,代码会使用pconn.addTLS(ctx, cm.tlsHost(), trace) 重新以普通方式进行握手,导致JA3修改失败
2. 因为golang强关联,第三方库并不能完美的集成到现有代码中,都需要重构代码
3. 某些网站会对于新建链接进行ClientSession检测,因此需要 KeepAlive+ClientSessionCache,这样通过复用连接减少风控概率

最终实现

1. 只需要拿到合法的参数,并且配置到TLSClientConfig里即可
2. 使用github.com/refraction-networking/utls中的UTLSIdToSpec拿到CipherSuites并传入
3. 因为有cipherSuitesPreferenceOrder,导致最终的指纹会有一定区别
4. 当然 也可以编写一个Ja3RoundTripper用于Transport

package main
import (
	"bytes"
	"crypto/tls"
	tlsx "github.com/refraction-networking/utls"
	"net/http"
)
func main() {
	c, _ := tlsx.UTLSIdToSpec(tlsx.HelloRandomized)
	a := &http.Client{
		Transport: &http.Transport{
			DisableKeepAlives: false,
			Proxy: proxy,
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
				MinVersion:         c.TLSVersMin,
				MaxVersion:         c.TLSVersMax,
				CipherSuites:       c.CipherSuites,
				ClientSessionCache: tls.NewLRUClientSessionCache(32),
			},
		},
	}
	aw, bw := a.Get("https://tls.browserleaks.com/json")
	defer aw.Body.Close()
	var buf bytes.Buffer
	aw.Write(&buf)
	println(string(buf.String()), bw)
}

参考文章

1. https://github.com/baixudong007/gospider
2. https://juejin.cn/post/7073264626506399751 用Go构建你专属的JA3指纹
3. https://blog.csdn.net/qq523176585/article/details/127116542 好库推荐|强烈推荐，支持Ja3指纹修改的golang请求库
4. https://github.com/wangluozhe/requests
5. https://github.com/refraction-networking/utls
6. http://www.ctfiot.com/64337.html 如何绕过 JA3 指纹校验？
7. https://www.coder.work/article/7192419 http - 发送请求时如何使用uTLS连接？
8. https://segmentfault.com/a/1190000041699815/en Build your own JA3 fingerprint with Go
9. https://zhuanlan.zhihu.com/p/601474166 curl_cffi: 支持原生模拟浏览器 TLS/JA3 指纹的 Python 库
10. https://tools.scrapfly.io/api/fp/ja3

历史编写的代码

这些代码都不太好用

package main

import (
	"context"
	"fmt"

	tls "github.com/refraction-networking/utls"
	xtls "github.com/refraction-networking/utls"
	"net"
	"net/http"
)

func GetTransport(helloID *xtls.ClientHelloID) *http.Transport {
	if helloID == nil {
		helloID = &xtls.HelloChrome_83
	}
	transport := http.DefaultTransport.(*http.Transport).Clone()
	transport.DialTLSContext = func(ctx context.Context, network, addr string) (_ net.Conn, err error) {
		dialer := net.Dialer{}
		con, err := dialer.DialContext(ctx, network, addr)
		if err != nil {
			return nil, err
		}
		// 根据地址获取host信息
		host, _, err := net.SplitHostPort(addr)
		if err != nil {
			return nil, err
		}
		c := transport.TLSClientConfig
		// 并且不验证host信息
		xtlsConf := &xtls.Config{
			ServerName: host,
			//Renegotiation:      xtls.RenegotiateNever,
			ClientSessionCache: xtls.NewLRUClientSessionCache(32),
			NextProtos:         []string{"h2", "http/1.1"},

			Rand: c.Rand,
			Time: c.Time,

			VerifyPeerCertificate: c.VerifyPeerCertificate,

			RootCAs: c.RootCAs,

			ClientCAs:                c.ClientCAs,
			InsecureSkipVerify:       c.InsecureSkipVerify,
			CipherSuites:             c.CipherSuites,
			PreferServerCipherSuites: c.PreferServerCipherSuites,
			SessionTicketsDisabled:   c.SessionTicketsDisabled,
			SessionTicketKey:         c.SessionTicketKey,

			MinVersion: c.MinVersion,
			MaxVersion: c.MaxVersion,

			DynamicRecordSizingDisabled: c.DynamicRecordSizingDisabled,

			KeyLogWriter: c.KeyLogWriter,
		}
		// 构建tls.UConn
		xtlsConn := xtls.UClient(con, xtlsConf, *helloID)

		// 握手
		err = xtlsConn.HandshakeContext(ctx)
		if err != nil {
			return nil, err
		}
		fmt.Println("当前请求使用协议：", xtlsConn.HandshakeState.ServerHello.AlpnProtocol)
		return xtlsConn, err
	}
	//transport.DialContext = transport.DialTLSContext
	transport.DisableKeepAlives = true

	return transport
}
func CreateHTTPClient() *http.Transport {
	return &http.Transport{
		DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {

			//initialize the tcp connection
			tcpConn, err := (&net.Dialer{}).DialContext(ctx, network, addr)
			if err != nil {
				return nil, err
			}
			host, _, err := net.SplitHostPort(addr)
			if err != nil {
				return nil, err
			}
			//initialize the conifg for tls
			config := tls.Config{
				ServerName:         host, //set the server name with the provided addr
				ClientSessionCache: xtls.NewLRUClientSessionCache(0),
			}

			//initialize a tls connection with the underlying tcop connection and config
			tlsConn := tls.UClient(tcpConn, &config, tls.HelloRandomized)

			//start the tls handshake between servers
			err = tlsConn.Handshake()
			if err != nil {
				return nil, fmt.Errorf("uTlsConn.Handshake() error: %w", err)
			}

			return tlsConn, nil
		},
		ForceAttemptHTTP2: false,
	}

}