代理ip
现在代理IP使用的是网站是蘑菇代理
API代理连接
在使用API代理的时候必须设置白名单。
heards = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'
}
proxies = {
'https':'ip:端口号' #'https':source['mag'][0]['ip']+':'+source['mag'][0]['port']
}
source = requests.get('网页',headers=headers,proxies=proxies).text
隧道代理连接
隧道ip个人觉得性价比比较高
#蘑菇代理的隧道订单
appKey = "直接复制粘贴"
#蘑菇隧道代理服务器地址
ip_port = 'secondtransfer.moguproxy.com:9001'
proxy = {"http": "http://" + ip_port,"https": "https://" + ip_port}
headers = {
"Proxy-Authorization": 'Basic '+ appKey,
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"}
source = requests.get("爬取的网站域名", headers=headers, proxies=proxy,verify=False,allow_redirects=False).text
demo = etree.HTML(source).xpath('copy的xpath')
多进程爬取
此项目爬取的是安居客二手房房源为例:
使用多进程进行爬取信息将会提高效率,使爬取数据变得更快,因为单进程爬取数据在需要爬取很多数据的时候效率非常的慢,会浪费很多的的时间所以使用多进程进行爬取。
import requests
from lxml import etree
from multiprocessing import Pool
#蘑菇代理的隧道订单
appKey = "OU5PN2hhaTEzRXB2ZkVFYjpBaW13cVhOdEV6bnZaSHU4"
#蘑菇隧道代理服务器地址
ip_port = 'secondtransfer.moguproxy.com:9001'
proxy = {"http": "http://" + ip_port,"https": "https://" + ip_port}
headers = {
"Proxy-Authorization": 'Basic '+ appKey,
"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4"}
def index(page):
source = requests.get('https://tianjin.anjuke.com/sale/p' + str(page) + '/?from=navigation',headers=headers, proxies=proxy,verify=False,allow_redirects=False).text
demo = etree.HTML(source).xpath(
'//*[@id="__layout"]/div/section/section[3]/section[1]/section[2]/div/a/div[2]/div[1]/div[1]/h3/text()')
print(demo)
print('====================当前在第'+str(page)+'页==========================')
if __name__=='__main__':
# print('Parent process %s.' % os.getpid())
p = Pool(8)
for page in range(1, 51):
p.apply_async(index, args=(page,))
print('Waiting for all subprocesses done...')
p.close()
p.join()