import requests
import bs4
import json
import time
import os
from lxml import etree
class spider(object):
def __init__(self):
self.url = "https://bj.zu.anjuke.com/?from=navigation"
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
def get_cities(self):
response = requests.get(self.url, headers=self.headers, timeout=30).text
html = etree.HTML(response)
city_list = html.xpath('//div[@id="city_list"]//dd/a')
cities = {}
for item in city_list:
city_url = item.xpath('./@href')[0]
city_name = item.xpath('./@title')[0]
cities[city_name] = city_url
if not os.path.exists('./data/' + city_name):
os.makedirs(&
爬虫实战20:爬取安居客租房信息
最新推荐文章于 2024-06-07 23:59:36 发布