阿宁的爬虫之旅----代理ip

代理ip

现在代理IP使用的是网站是蘑菇代理

API代理连接

在使用API代理的时候必须设置白名单。

heards = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'
}

proxies = {
    'https':'ip:端口号'   #'https':source['mag'][0]['ip']+':'+source['mag'][0]['port']
}
source = requests.get('网页',headers=headers,proxies=proxies).text

隧道代理连接

隧道ip个人觉得性价比比较高

#蘑菇代理的隧道订单
appKey = "直接复制粘贴"
#蘑菇隧道代理服务器地址
ip_port = 'secondtransfer.moguproxy.com:9001'
proxy = {"http": "http://" + ip_port,"https": "https://" + ip_port}
headers = {
  "Proxy-Authorization": 'Basic '+ appKey,
  "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0",
  "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"}
source = requests.get("爬取的网站域名", headers=headers, proxies=proxy,verify=False,allow_redirects=False).text
demo = etree.HTML(source).xpath('copy的xpath')

多进程爬取

此项目爬取的是安居客二手房房源为例：

使用多进程进行爬取信息将会提高效率，使爬取数据变得更快，因为单进程爬取数据在需要爬取很多数据的时候效率非常的慢，会浪费很多的的时间所以使用多进程进行爬取。

import requests
from lxml import etree
from multiprocessing import Pool
#蘑菇代理的隧道订单
appKey = "OU5PN2hhaTEzRXB2ZkVFYjpBaW13cVhOdEV6bnZaSHU4"

#蘑菇隧道代理服务器地址
ip_port = 'secondtransfer.moguproxy.com:9001'
proxy = {"http": "http://" + ip_port,"https": "https://" + ip_port}
headers = {
  "Proxy-Authorization": 'Basic '+ appKey,
  "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0",
  "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"}

def index(page):
    source = requests.get('https://tianjin.anjuke.com/sale/p' + str(page) + '/?from=navigation',headers=headers, proxies=proxy,verify=False,allow_redirects=False).text
    demo = etree.HTML(source).xpath(
        '//*[@id="__layout"]/div/section/section[3]/section[1]/section[2]/div/a/div[2]/div[1]/div[1]/h3/text()')
    print(demo)
    print('====================当前在第'+str(page)+'页==========================')
if __name__=='__main__':
    # print('Parent process %s.' % os.getpid())
    p = Pool(8)
    for page in range(1, 51):
        p.apply_async(index, args=(page,))
    print('Waiting for all subprocesses done...')
    p.close()
    p.join()