Python通过代理使用多线程爬取安居客二手房数据（二）

在上一篇文章python使用代理爬取安居客二手房数据（一）上增加爬取内容
爬取内容为：：‘待售房屋’, ‘室’, ‘厅’, ‘卫’, ‘面积’, ‘面积单位’, ‘朝向’, ‘楼层’, ‘建筑年份’,
‘小区名称’, ‘区’, ‘镇’, ‘道路’, ‘标签’, ‘总价’, ‘总价单位’, ‘均价’, ‘均价单位’ 并使用多线程提高爬取速度

爬取网址管理器

"""
@author rubyw
爬虫的url管理器
"""


class CrawlerUrlManager():

    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()

    # 新增一个待爬取Url
    def add_new_url(self, url):
        if url is None or len(url) == 0:
            return
        if url in self.new_urls or url in self.old_urls:
            return
        self.new_urls.add(url)
        return True

    # 批量新增url
    def add_new_urls(self, urls):
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)

    # 获取一个要爬取的url
    def get_url(self):
        if self.has_new_url():
            url = self.new_urls.pop()
            self.old_urls.add(url)
            return url
        else:
            return None

    # 批量获取待爬取的url
    def get_new_urls(self, num):
        returnUrls = set()
        if num is None or type(num) != int or num <= 0:
            return returnUrls
        else:
            i = 0
            while self.has_new_url() and i < num:
                url = self.new_urls.pop()
                self.old_urls.add(url)
                returnUrls.add(url)
                i = i + 1
        return returnUrls

    # 判断是否有待爬取的url
    def has_new_url(self):
        return len(self.new_urls) > 0

    # 获取待爬取url的数量
    def get_new_url_size(self):
        return len(self.new_urls)

    # 获取已爬取url的数量
    def get_old_url_size(self):
        return len(self.old_urls)


if __name__ == "__main__":
    url_manager = CrawlerUrlManager()

    # 添加两个url，批量添加故意添加一个重复的url，看去重是否ok
    url_manager.add_new_url("url1")
    url_manager.add_new_urls(["url1", "url2"])
    print(url_manager.new_urls, url_manager.old_urls)

    # 获取一个url，然后打印两个集合
    print("#" * 30)
    new_url = url_manager.get_url()
    print(url_manager.new_urls, url_manager.old_urls)

    # 再获取一个url，然后打印两个集合
    print("#" * 30)
    new_url = url_manager.get_url()
    print(url_manager.new_urls, url_manager.old_urls)

    # 看看两个集合中还有没有Url
    print("#" * 30)
    print(url_manager.has_new_url())

爬取二手房数据

"""
爬取安居客网站苏州的二手房数据
启动方法：进入/anjuke目录下，执行python secondhand_house_crawler.py
"""

import requests
from bs4 import BeautifulSoup
import threading
import time
import csv
from crawlUrlManager import CrawlerUrlManager


def get_proxies():
    proxy_list = []
    proxy_url = ''  # 替换成自己的
    try:
        datas = requests.get(proxy_url).json()
        print(datas['code'])
        # 如果代理ip获取成功
        if datas['code'] == 0:
            proxy_list = datas['data']['proxy_list']
            # data_array = datas['data']
            # for i in range(len(data_array)):
            #     proxy_ip = data_array[i]['ip']
            #     proxy_port = str(data_array[i]['port'])
            #     proxy = proxy_ip + ":" + proxy_port
            #     proxy_list.append({'http': 'http://' + proxy, 'https': 'http://' + proxy})
        else:
            code = datas['code']
            print(f'获取代理失败，状态码={code}')
        return proxy_list
    except Exception as e:
        # print('调用天启API获取代理IP异常:' + e)
        print('调用快代理API获取代理IP异常:' + e)
        return proxy_list


def craw_anjuke_wuhan(craw_url, proxy):
    if craw_url is None:
        print(threading.current_thread().getName() + ' craw_url is None')
        return

    # 用户名密码认证(私密代理/独享代理)
    username = ""  # 替换成自己的
    password = ""  # 替换成自己的

    proxies = {
        "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {'user': username, 'pwd': password,
                                                        'proxy': proxy},
        "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {'user': username, 'pwd': password,
                                                         'proxy': proxy}
    }

    print(proxies)

    print(threading.current_thread().getName() + f' is crawing {craw_url}...使用代理{proxy}')

    # 构造url的request headers，伪装成正常用户
    headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'zh-CN,zh;q=0.9',
        'cache-control': 'no-cache',
        'cookie': 'aQQ_ajkguid=70C0288A-42CB-4C56-B8EF-8E90F8077A8C; sessid=13C76F04-9178-4EE8-B8B0-F00FE21F4F50; ajk-appVersion=; ctid=22; fzq_h=d23302afd92c82b304657a734e3950aa_1697613588983_b645e9292cff4c148c0e3fb2ff31662e_3746354997; id58=CrIej2Uvhxc/D8k8IRI2Ag==; twe=2; fzq_js_anjuke_ershoufang_pc=8e86fa86290dbac07d5de51dd3b9db13_1697615100824_23; obtain_by=1; xxzl_cid=817f908b661647889fa49debaab80d9c; xxzl_deviceid=lrdQ4FRXrfXyN2Qj/gRhBw2SQpTZ81igKeOBCkzlfzjPwEG8whpE1uKNvVqIOvXQ',
        'host': 'wuhan.anjuke.com',
        'pragma': 'no-cache',
        'referer': 'https://wuhan.anjuke.com/sale/jianghana/p1/',
        'sec-ch-ua': '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': "Windows",
        'sec-fetch-dest': 'document',
        'sec-fetch-mode': 'navigate',
        'sec-fetch-site': 'same-origin',
        'sec-fetch-user': '?1',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36'
    }

    with open('data/wuhanSecondHouse.csv', 'a', newline='', encoding="gbk") as f:
        # 有代理用代理，没代理直接爬
        if proxy is None:
            r = requests.get(craw_url, headers=headers, timeout=3)
        else:
            r = requests.get(craw_url, headers=headers, proxies=proxies, timeout=3)
        # 如果正常返回结果，开始解析
        if r.status_code == 200:
            content = r.text
            # print(content)
            soup = BeautifulSoup(content, 'html.parser')
            content_div_nodes = soup.find_all('div', class_='property-content')
            for content_div_node in content_div_nodes:
                # 获取房产标题内容
                content_title_name = content_div_node.find('h3', class_='property-content-title-name')
                title_name = content_title_name.get_text()
                # 获取房子户型
                content_layout = content_div_node.find('p',
                                                       class_='property-content-info-text property-content-info-attribute')
                layout_datas = content_layout.find_all('span')
                datas_shi = layout_datas[0].get_text() + layout_datas[1].get_text()
                datas_ting = layout_datas[2].get_text() + layout_datas[3].get_text()
                datas_wei = layout_datas[4].get_text() + layout_datas[5].get_text()
                # 获取房子的面积、朝向、楼层和建筑年份
                square_num = ''
                square_unit = ''
                orientations = ''
                floor_level = ''
                build_year = ''
                content_extra_info_datas = content_div_node.find_all(
                    lambda content_div_node: content_div_node.name == 'p' and content_div_node.get('class') == [
                        'property-content-info-text'])
                for i in range(len(content_extra_info_datas)):
                    if i == 0:
                        square = content_extra_info_datas[0].get_text().strip()
                        square_num = square[0:len(square) - 1]
                        square_unit = square[len(square) - 1:]
                    if i == 1:
                        orientations = content_extra_info_datas[1].get_text().strip()
                    if i == 2:
                        floor_level = content_extra_info_datas[2].get_text().strip()
                    if i == 3:
                        build_year = content_extra_info_datas[3].get_text().strip()
                # 获取房子的小区名称、位置信息（区-镇-道路）
                content_info_comm = content_div_node.find('div',
                                                          class_='property-content-info property-content-info-comm')
                # 获取小区名称
                housing_estate = content_info_comm.find('p',
                                                        class_='property-content-info-comm-name').get_text().strip()
                # 获取小区地址信息
                content_info_address = content_info_comm.find('p',
                                                              class_='property-content-info-comm-address').find_all(
                    'span')
                district = content_info_address[0].get_text().strip()
                town = content_info_address[1].get_text().strip()
                road = content_info_address[2].get_text().strip()
                # 获取房子的更多tag信息，比如朝向、是否满五唯一、房子新旧、是否近地铁等
                content_info_tag = content_div_node.find_all('span', class_='property-content-info-tag')
                tagstr = ''
                for i in range(len(content_info_tag)):
                    tagstr = tagstr + content_info_tag[i].get_text().strip() + ','
                # 获取房子价格信息
                price_info_datas = content_div_node.find('div', class_='property-price')
                total_price = price_info_datas.find('span', class_='property-price-total-num').get_text().strip()
                total_price_unit = price_info_datas.find('span', class_='property-price-total-text').get_text().strip()
                avarage_price = price_info_datas.find('p', class_='property-price-average').get_text().strip()
                avarage_price_num = avarage_price[0:len(avarage_price) - 3]
                avarage_price_unit = avarage_price[len(avarage_price) - 3:]
                # 输出到文件
                writer = csv.writer(f)
                writer.writerow([title_name, datas_shi, datas_ting, datas_wei, square_num, square_unit, orientations, floor_level,
                build_year, housing_estate, district, town, road, tagstr, total_price, total_price_unit,
                avarage_price_num, avarage_price_unit])
                # f.write("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s\n" % (
                # title_name, datas_shi, datas_ting, datas_wei, square_num, square_unit, orientations, floor_level,
                # build_year, housing_estate, district, town, road, tagstr, total_price, total_price_unit,
                # avarage_price_num, avarage_price_unit))
            print(f'{threading.current_thread().getName()} crawl over!;Crawler Url is:{craw_url}')
        else:
            print(
                f'{threading.current_thread().getName()} crawl fail!status code={r.status_code};Crawler Url is:{craw_url}')


if __name__ == '__main__':

    # 先将标题写入结果数据文件
    with open('data/wuhanSecondHouse.csv', 'w', newline='', encoding='gbk') as f:
        writer = csv.writer(f)
        writer.writerow(
        ['待售房屋', '室', '厅', '卫', '面积', '面积单位', '朝向', '楼层', '建筑年份', '小区名称', '区', '镇', '道路', '标签', '总价', '总价单位', '均价',
        '均价单位']
        )

    # 假设爬取crawler_pages页，生成待爬取的url，放入url池管理起来
    crawlerUrlManager = CrawlerUrlManager()
    # 要爬取的页数，默认为100，可调整
    crawler_pages = 200
    for i in range(crawler_pages):
        region = "jianghana"
        url = 'https://wuhan.anjuke.com/sale/{region}/p{page}/'
        craw_url = url.format(region=region, page=i)
        crawlerUrlManager.add_new_url(craw_url)

    # 尝试获取代理ip，避免同一个ip频繁访问被网站的反爬机制给封禁
    proxy_list = get_proxies()
    proxy_num = len(proxy_list)
    if proxy_num >= 2:  # 如果获取到代理ip，则用代理ip，建议至少获取5个及以上的代理ip，爬取的时候每个线程一个ip进行爬取
        print(f'获取到{proxy_num}个代理ip，开始使用代理IP爬取页面数据...')

        while crawlerUrlManager.has_new_url():
            crawler_threads = []
            for i in range(len(proxy_list)):
                proxy = proxy_list[i]
                print(crawlerUrlManager.get_url())
                crawler_thread = threading.Thread(craw_anjuke_wuhan(crawlerUrlManager.get_url(), proxy))
                crawler_threads.append(crawler_thread)

            # 启动线程开始爬取
            for crawler_thread in crawler_threads:
                crawler_thread.start()

            for crawler_thread in crawler_threads:
                crawler_thread.join()

            # 谨慎起见，一批线程爬取结束后，间隔一段时间，再启动下一批爬取，这里默认设置为3秒，可调整
            time.sleep(3)

    else:  # 如果没获取到代理ip，则直接爬取，控制一下每个线程爬取的间隔时间，不要太频繁
        try:
            print('没有获取到代理IP，开始使用自身IP爬取页面数据...')
            while crawlerUrlManager.has_new_url():
                crawler_thread = threading.Thread(target=craw_anjuke_wuhan, args=(crawlerUrlManager.get_url(), None))
                crawler_thread.start()

                crawler_thread.join()
                time.sleep(10)  # 为避免同一个ip频繁爬取被反爬封禁，一线程爬取完后，等待10秒再爬取下一个页面

        except Exception as e:
            print('Crawler Excepiton:' + e)
        finally:
            print(f'已爬取的url数量：{crawlerUrlManager.get_old_url_size()}')
            print(f'未爬取的url数量：{+crawlerUrlManager.get_new_url_size()}')
            if crawlerUrlManager.get_new_url_size() > 0:
                print('未爬取的url如下：')
                for new_url in crawlerUrlManager.get_url():
                    print(f'{new_url}')