import requests
from lxml import etree
#import time
from selenium import webdriver
import pandas as pd
from bs4 import BeautifulSoup
#需要将chromedriver放到Chrome\Application目录下
#chrome_driver=r"C:\Program Files (x86)\Google\Chrome\Application\chromedriver_win32\chromedriver.exe"
chrome_driver=“C:\Program Files (x86)\Google\Chrome\Application\chromedriver_win32\chromedriver.exe”
driver = webdriver.Chrome(executable_path=chrome_driver)
request_url = ‘https://beijing.anjuke.com/sale/p’
print(html)
html = etree.HTML(html)
#print(html)
#去空格,去换行\n
def format_str(str):
return str.replace(’\n’, ‘’).replace(’ ', ‘’)
houses = pd.DataFrame(columns=[“name”, “details”, “address”, “price”])
for i in range(3):
url = request_url + str(i+1)
driver.get(url)
html = driver.find_element_by_xpath("//*").get_attribute("outerHTML")
soup = BeautifulSoup(html, "html.parser", from_encoding="utf-8")
house_list = soup.find_all("li", class_="list-item")
for house in house_list:
temp = {}
#提取房源信息
name = house.find("div", class_="house-title").a.text.strip()
details = house.find("div", class_="details-item").text.strip()
address = house.find("span", class_="comm-address").text.strip()
price = house.find("span", class_="price-det").text.strip()
# print("name:{} detai:{} address:{} price:{}".format(name, details, address, price))
temp["name"] = format_str(name)
temp["details"] = format_str(details)
temp["address"] = format_str(address)
temp["price"] = format_str(price)
houses = houses.append(temp, ignore_index=True)
#to_csv 保存csv文件
houses.to_csv(“beijing.csv”, index=False,encoding=‘utf_8_sig’)