目标
1、安居客二手房,随便一个城市,选择优秀经纪人,爬取所有页数
网址:http://beijing.anjuke.com/tycoon/p/
过程
1、在网页源代码中找到所需要的数据
2、用bs提取出所需要的信息,代码如下
#encoding=utf8
import re
import urllib
import urllib2
from bs4 import BeautifulSoup #导入bs
url = 'http://beijing.anjuke.com/tycoon/p'
for page in range(1,11):
url1 = url+str(page)+'/'
print url1
# 不传入headers的话无法访问
user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0"
headers = {"User-Agent": user_agent}
request = urllib2.Request(url1, headers=headers)
response = urllib2.urlopen(request)
html = response.read()
# pages = urllib.urlopen(url)
# html = pages.read()
soup = BeautifulSoup(html, 'html.parser')
# print html
# print(soup.prettify())
title = soup.find_all('div',class_='jjr-itemmod') #所需的信息在这个大的div标签内
# print title
for til in title:
part1 = til.find('div',class_='jjr-info').get_text('|',strip=True).encode('utf-8')
#一个标签里包含了多条数据,用|分隔开
part3 = part1.replace(' ','')
part4 = part3.replace('\n','')
part2 = til.find('div',class_='jjr-side').get_text(strip=True).encode('utf-8')
#去掉空格回车等不需要的东西
# print '-->',part4,'<--' #查看取出的内容
with open('anjuke.txt','a+') as AJ:
AJ.write(part4+'|'+part2+'\n')