链家房价信息爬虫

朋友准备买房子，我帮着爬取了一下链家的房子信息。python版本为3.6，使用openpyxl写入excel。在clears = lj.find_all(‘li’, attrs={‘class’: ‘clear’})中，’li’和’class’: ‘clear’时该房子连接的信息（鼠标放在

• 上会选中该房子的全部信息）。

• 爬虫

  # 链家网二手房信息爬取
  import re
  import time
  import requests
  import pandas as pd
  from bs4 import BeautifulSoup
  from openpyxl import Workbook
  
  url = 'https://bj.lianjia.com/ershoufang/'
  page = ('pg')
  # 设置请求头部信息
  headers = {
      'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
      'Accept': 'text/html;q=0.9,*/*;q=0.8',
      'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
      'Accept-Encoding': 'gzip',
      'Connection': 'close',
      'Referer': 'http://www.baidu.com/link?url=_andhfsjjjKRgEWkj7i9cFmYYGsisrnm2A-TN3XZDQXxvGsM9k9ZZSnikW2Yds4s&wd=&eqid=c3435a7d00146bd600000003582bfd1f'
      }
  
  # 翻页
  for i in range(1, 101):
      if i == 1:
          i = str(i)
          a = (url + page + i + '/')
          r = requests.get(url=a, headers=headers)
          html = r.content
      else:
          i = str(i)
          a = (url + page + i + '/')
          r = requests.get(url=a, headers=headers)
          html2 = r.content
          html = html + html2
      # 每次间隔0.5秒
      time.sleep(0.5)
  
  # 解析抓取的页面内容
  lj = BeautifulSoup(html, 'html.parser')
  
  clears = lj.find_all('li', attrs={'class': 'clear'})
  houseInfo = []
  guanzhuInfo = []
  daikanInfo = []
  timeInfo = []
  subwayInfo = []
  positionInfo = []
  totalpriceInfo = []
  
  #将信息加入对应的列表中
  for clear in clears:
      houseInfo.append(clear.find('div', 'houseInfo').get_text())
  
      guanzhuInfo.append(clear.find(text=re.compile('人关注')))  
      daikanInfo.append(clear.find(text=re.compile('次带看')))
  
      timeInfo.append(clear.find('div', 'timeInfo').get_text())
  
      temp = clear.find('span', 'subway')
      if temp is None:
          subwayInfo.append('')
      else:
          subwayInfo.append(temp.get_text())
  
      positionInfo.append(clear.find('div', 'positionInfo').get_text())
      totalpriceInfo.append(clear.find('div', 'totalPrice').get_text())
  
  
  pd.set_option('display.max_colwidth',5000)
  
  data ={
      'houseInfo': houseInfo,
      'guanzhuInfo': guanzhuInfo, 'daikanInfo': daikanInfo,
      'timeInfo': timeInfo, 'subwayInfo': subwayInfo, 'positionInfo': positionInfo, 'totalpriceInfo': totalpriceInfo
  }
  #创建excel
  wb = Workbook()
  sheet = wb.active
  #创建一个新的sheet
  sheet.title = "New Sheet"
  #将列表传入excel对应的行中
  sheet.append(houseInfo)
  sheet.append(guanzhuInfo)
  sheet.append(daikanInfo)
  sheet.append(timeInfo)
  sheet.append(subwayInfo)
  sheet.append(positionInfo)
  sheet.append(totalpriceInfo)
  
  #保存excel
  wb.save('链家房子信息.xlsx')
  
  print('finish!')