【爬虫专栏19】爬取链家网（仅小区IP）

import re 
import requests
from lxml import etree
import csv
import time
import json
from fake_useragent import UserAgent


#单线程抓取小区id前100页信息
def get_xiaoqu(x):
    head = {'Host': 'bj.lianjia.com',
            'Referer': 'https://bj.lianjia.com/chengjiao/',
            "User-Agent" : UserAgent().chrome #chrome浏览器随机代理
            #'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'
            }
    n=x*20+1
    l=list(range(n,n+20))
    for i in l:
        url = 'https://bj.lianjia.com/xiaoqu/pg' + str(i)
        try:
            r = requests.get(url, headers=head, timeout=3)
            html = etree.HTML(r.text)
            datas=html.xpath('//li[@class="clear xiaoquListItem"]/@data-id')
            title=html.xpath('//li[@class="clear xiaoquListItem"]/div[@class="info"]/div[@class="title"]/a/text()')
            print('No:' + str(x), 'page:' + str(i), len(s), len(datas), len(title))
            #如果当前页没有返回数据，将当前页数加到列表l末尾，再次抓取
            if len(datas)==0:
                print(url)
                l.append(i)
            else:
                for data in datas:
                    s.add(data)
        # 如果当前页访问出现异常，将当前页数加到列表l末尾，再次抓取
        except Exception as e:
            l.append(i)
            print(e)
 
    print('      ****No:'+str(x)+' finish')
#本人购买的代理获取方式，需要根据你们自己的修改。函数功能获取n个ip，并以列表形式返回，每个元素为字典：{'https':'https://118.120.228.202:4286'}
def get_ip(n):
    url='XXXXXXXXXXXXXXXXXXXXXXXX'
    r=requests.get(url)
    html=json.loads(r.text)
    proxies=[]
    for i in range(n):
        a=html['data'][i]['ip']
        b=html['data'][i]['port']
        val='https://'+str(a)+':'+str(b)
        p={'https':val}
        proxies.append(p)
    return(proxies)

'''
#IP代理
ips = [] #装载有效 IP 
for i in range(1, 6):
    headers = {
    "User-Agent" : UserAgent().chrome #chrome浏览器随机代理
    }
    ip_url = 'http://www.89ip.cn/index_{}.html'.format(i)
    html = requests.get(url=ip_url, headers=headers).text
    res_re = html.replace(" ", "").replace("\n", "").replace("\t", "")
    #使用正则表达式匹配出IP地址及端口
    r = re.compile('<tr><td>(.*?)</td><td>(.*?)</td><td>')
    result = re.findall(r, res_re)
    for i in range(len(result)):
        ip = "http://" + result[i][0] + ":" + result[i][1]
        # 设置为字典格式
        proxies = {"http": ip}
        #使用上面的IP代理请求百度，成功后状态码200
        baidu = requests.get("https://www.baidu.com/", proxies = proxies)
        if baidu.status_code == 200:        
            ips.append(proxies)
    print ("正在准备IP代理，请稍后。。。")
'''
    
     
if __name__=='__main__':
    global s
    #将id保存在set中，达到排重效果
    s = set()
    #该页面网站会禁ip，所以每个ip只访问20页
    for x in range(2):
        now=time.time()
        #ls = get_ip(1)
        #p=ls[0]
        get_xiaoqu(x)
        print(time.time()-now)
    print('******************')
    print('抓取完成')
    #将抓取的id保存到本地csv
    with open(r'E:\vscode_code\多线程\链家网\xiaoqu_id.csv', 'a', newline='', encoding='gb18030')as f:
        write = csv.writer(f)
        for data in s:
            write.writerow([data])
        f.close()

根据小区IP就可以确定每个小区的所有房子，访问起来更加方便