python 爬虫 booking爬取酒店信息

import requests
from bs4 import BeautifulSoup as bs
import re
import time
import pandas as pd


headers ={"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
          "Cookie":"设置好地区 货币等信息后的cookie"}

url_list=["用列表 好让爬虫自动遍历列表翻页。这里输入的url是在页面上选好地区 货币等信息后的url 这点一定注意！"]
info =[]
for ii in url_list:
    time.sleep(2)
    response = requests.get(ii,headers=headers)
    #print(response.text)

    #提取单个酒店信息代码块
    soup = bs(response.text,"lxml")
    hotle_info = soup.select("#hotellist_inner div.sr_item_content.sr_item_content_slider_wrapper")
    #print(len(hotle_info))
    #print(hotle_info)

    #提取下一页链接
    try:
        next_page = soup.select("#search_results_table div.bui-pagination.results-paging div.bui-pagination__nav ul li.bui-pagination__item.bui-pagination__next-arrow a")
        s5 = r'href="(.*?)" title='
        next_page_halfurl =re.compile(s5,re.S).findall(str(next_page))[0]
        #print(next_page_halfurl)
    except:
        break


    #组合完整链接 并且放入队列 让其持续翻页
    next_page_allurl = "https://www.booking.com"+ next_page_halfurl
    url_list.append(next_page_allurl)


    for i in hotle_info:
        #每一个酒店的价格
        s1 = r'<b class=" ">\n¥(.*?)\n</b>'
        price = re.compile(s1,re.S).findall(str(i))
        #print("".join(price)) #去壳方法 "".join()  空气join(内容)
        price_info = ("".join(price))

        #酒店的链接
        s2 = r'href="\n(.*?)" rel="noopener"'
        hotle_link =re.compile(s2,re.S).findall(str(i))[0]
        #print("https://www.booking.com" + hotle_link)
        hotle_link_info =("https://www.booking.com" + hotle_link)

        #酒店的名称
        s3 = r'sr-hotel__name " data-et-click="\n">\n(.*?)\n</span>'
        hotle_name = re.compile(s3,re.S).findall(str(i))
        #print("".join(hotle_name))
        hotle_name_info = ("".join(hotle_name))

        #酒店的区域
        s4 = r'title="旅友们喜爱\n(.*?)\n的理由'
        hotle_quyu = re.compile(s4,re.S).findall(str(i))
        #print("".join(hotle_quyu))
        hotle_quyu_info =("".join(hotle_quyu))

        hotle_info_info = []
        hotle_info_info.append(price_info)
        hotle_info_info.append(hotle_link_info)
        hotle_info_info.append(hotle_name_info)
        hotle_info_info.append(hotle_quyu_info)

        info.append(hotle_info_info)
        print(info)
#将信息制作成表格，便于查看对比
name = ["价格","链接","名称","区域"]
test = pd.DataFrame(columns=name,data=info)
test.to_csv("D:\酒店.csv")

转载于:https://www.cnblogs.com/cwkcwk/p/9576533.html