爬虫中用到的代理虽然可以隐藏ip,避免真实ip被封,但是用久了,代理ip也会被封掉,这个时候就需要用到代理池,所谓代理池,就是一系列的代理,代码中可以随机选取代理池的任意代理访问目的服务器,从而避免同一ip高频次访问目的服务器。
代码:
import random
import urllib.request
proxies_pool = [
{'http':'118.24.219.151:1681711111'},
{'http':'118.24.219.151:1681722222'}
]
proxies = random.choice(proxies_pool)
print(proxies)
url = 'http://www.baidu.com/s?wd=ip'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36',
'Cookie': 'BIDUPSID=F352E19EDFD18E88A6D111AE7CB7B3A8; PSTM=1611757517; __yjs_duid=1_6cc240ffe6476ab065efda6b1be0a0641621078616582; BDUSS=JPVHdLbkFsVGRiRWdCfjAwVmgyTi1XOTFVRE1mQTJ5T3dpOEtJcmljYy1oaU5oSUFBQUFBJCQAAAAAAAAAAAEAAACTSWqlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD75-2A--ftgSm; BDUSS_BFESS=JPVHdLbkFsVGRiRWdCfjAwVmgyTi1XOTFVRE1mQTJ5T3dpOEtJcmljYy1oaU5oSUFBQUFBJCQAAAAAAAAAAAEAAACTSWqlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD75-2A--ftgSm; BDSFRCVID=5DuOJeC62uh-in5epD52bcgumdlrgK3TH6aoVtIvkPJstU3I3eVnEG0PfM8g0KubJmfpogKKKgOTHICF_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=JRkH_IIhtCvbfP0k-tcH244HqxbXqMJQW2OZ0l8KtDQjeU3FK4od5q09XfRUq4rMKC-q3brmWIQHDU7b5PTF5U-tqJQPB53IbK54KKJx3pCWeIJo5DcB2hFvhUJiBh-HBan7-pvIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtpChbC_6DTKKejb0eUcMetnKa55KWD8KJ6rjDCklDURcy4LdjG5f35J0-D7-Wbn92l6OSP5C0bJCqf_q3-Aq54RzaejyKxQtBKQOEUjDMTK-QfbQ0MnuqP-jW26a2J7J0b7JOpv5Dxnxyh0UQRPH-Rv92DQMVU52QqcqEIQHQT3m5-5bbN3ht6IHtnFJVIt5tDvKeJbYK4oj5KCyMfca5C6JKCOa3RA8Kb7VbI5e5fnkbfJBDlo3e4cXJCTjKIK-KJOJfJ6ghb5JyTt7yajK2h5JW6cWbpcbbUK5ODON34TpQT8rb4FOK5OibCrZWRQMab3vOpvTXpO1yftzBN5thURB2DkO-4bCWJ5TMl5jDh3Mb6ksD-FtqtJHKbDH_D0XtMK; BAIDUID=7C02F8439BAFCCDD1BCC649AEE87C0AD:FG=1; BDSFRCVID_BFESS=5DuOJeC62uh-in5epD52bcgumdlrgK3TH6aoVtIvkPJstU3I3eVnEG0PfM8g0KubJmfpogKKKgOTHICF_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=JRkH_IIhtCvbfP0k-tcH244HqxbXqMJQW2OZ0l8KtDQjeU3FK4od5q09XfRUq4rMKC-q3brmWIQHDU7b5PTF5U-tqJQPB53IbK54KKJx3pCWeIJo5DcB2hFvhUJiBh-HBan7-pvIXKohJh7FM4tW3J0ZyxomtfQxtNRJ0DnjtpChbC_6DTKKejb0eUcMetnKa55KWD8KJ6rjDCklDURcy4LdjG5f35J0-D7-Wbn92l6OSP5C0bJCqf_q3-Aq54RzaejyKxQtBKQOEUjDMTK-QfbQ0MnuqP-jW26a2J7J0b7JOpv5Dxnxyh0UQRPH-Rv92DQMVU52QqcqEIQHQT3m5-5bbN3ht6IHtnFJVIt5tDvKeJbYK4oj5KCyMfca5C6JKCOa3RA8Kb7VbI5e5fnkbfJBDlo3e4cXJCTjKIK-KJOJfJ6ghb5JyTt7yajK2h5JW6cWbpcbbUK5ODON34TpQT8rb4FOK5OibCrZWRQMab3vOpvTXpO1yftzBN5thURB2DkO-4bCWJ5TMl5jDh3Mb6ksD-FtqtJHKbDH_D0XtMK; BDRCVFR[fb3VbsUruOn]=ddONZc2bo5mfAF9pywdpAqVuNqsus; delPer=0; BD_CK_SAM=1; BAIDUID_BFESS=14ABB9F98198351F36AECB73A27BCCDC:FG=1; channel=baidusearch; BD_HOME=1; ZD_ENTRY=bing; BD_UPN=12314753; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; baikeVisitId=478366d6-46de-46d8-8708-1e881e26dc5b; ab_sr=1.0.1_NTJmMTFmYWU4YmRjZDE3ZGIyMmQ4MWI3NzlhYWY4ODM3ODRmNTk4MzNhY2NmOTFkYWQ1MTcxNWQwNTM3MjQzMGM4M2Q1MTFiNWUzNDNkYjEyZTQ2MjZiMWQ1N2QyNzhmYmUyOTRlNDE2ZTAzMDM5Y2ZiNTNiOWMwMTI4OWM5N2M3NDhhMGUyODI2ZjgwNjIyNTVlOGVkMzgzYWI2ODZhNzE1NTE0NmUwNGQ2ZTIzNzAzMDNiZDU4OGQwMGFmOTZh; PSINO=1; H_PS_PSSID=34430_35104_31660_35488_34584_35490_35318_26350_35723_22158; BDRCVFR[feWj1Vr5u3D]=mk3SLVN4HKm; H_PS_645EC=8d3chTqzQmY5UjJhkHfk8cx%2F67mzcub3Ny5hHXVm%2BPdytPVpBmaft4Khn7hKgqRg4vqu; BA_HECTOR=a02ga4ak2g21aha4ko1h06dn50r; COOKIE_SESSION=3_2_5_4_4_3_1_0_5_3_0_0_26378412_0_0_0_1643118733_1627196230_1644377829%7C9%2311047512_4_1627196228%7C3'
}
request = urllib.request.Request(url = url, headers = headers)
handler = urllib.request.ProxyHandler(proxies = proxies)
opener = urllib.request.build_opener(handler)
response = opener.open(request)
content = response.read().decode('utf-8')
with open('daili.html', 'w', encoding= 'utf-8') as fp:
fp.write(content)