python-爬虫-到喜啦酒店信息抓取

到喜啦 酒店信息抓取

1.招商需要成都的酒店信息，那么时间比较短
上代码

import requests
import pymysql
from lxml import etree
import re

class DXL:
    def __init__(self):
        self.headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            'Cookie': '_da=DA.234542667.1565251587567; city_id=144; city=cd; _ga=GA1.2.1039941117.1565251588; PHPSESSID=SqH33WnW595aGEfx-wV8hZT0Cw; _uab_collina=156525163408777721466634; _gid=GA1.2.1336936516.1565318781; utm_source=seo; utm_medium=pc_baidu_seo; utm_campaign=; utm_term=%E5%88%B0%E5%96%9C%E5%95%A6; utm_content=; a_i=; _ba=778128998647663; Hm_lvt_1d2519efe52fbcddc471e1b2ee80eb9e=1565251588,1565318781,1565332693,1565335299; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2216c7043d0a58d8-0b70f68a79097c-3c375f0d-2073600-16c7043d0a7aab%22%2C%22%24device_id%22%3A%2216c7043d0a58d8-0b70f68a79097c-3c375f0d-2073600-16c7043d0a7aab%22%2C%22props%22%3A%7B%22%24latest_referrer%22%3A%22https%3A%2F%2Fsp0.baidu.com%2F9q9JcDHa2gU2pMbgoY3K%2Fadrc.php%3Ft%3D06KL00c00f7-cK30swF-0QSDcsjeZbdI00000KGDr-C00000V3ZsCC.THvdETSzsQc0UWdBmy-bIy9EUyNxTAT0T1Y3rHckuh7huH0snAf4rAfs0ZRqwbn3PW9afYnkPH6vrjRLn1c1nYnsnbR%22%2C%22%24latest_referrer_host%22%3A%22sp0.baidu.com%22%2C%22%24latest_traffic_source_type%22%3A%22%E5%BC%95%E8%8D%90%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC%22%7D%7D; Hm_lpvt_1d2519efe52fbcddc471e1b2ee80eb9e=1565335886',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
        }
        self.url = 'https://cd.daoxila.com/HunYan/'
        self.db = pymysql.connect(host='172.16.201.20', port=3306, user='xiaoshubiao', passwd='654321', db='sina',
                                  charset='utf8mb4')
        self.cursor = self.db.cursor()

    def shop_mess(self):
        Area = ['WuHou', 'JinNiu', 'QingYang', 'JinJiang',
                'ChengDuGaoXin', 'ChengHua', 'ChengDuJinJiao','HuaYang', 'XinDu', 'WenJiang', 'ShuangLiuQu', 'LongQuan', 'DuJiangYan']
        for A in Area:
            url = self.url + A
            for i in range(1, 11):
                fin_url = url + '/page-' + str(i)
                print(fin_url)
                response = requests.get(fin_url, headers=self.headers)
                page = etree.HTML(response.text)
                href = page.xpath('//*[@id="hotelView"]/li/article/div/div[1]/h2/a/@href')
                area = page.xpath('//*[@id="hotelView"]/li/article/div/div[2]/i[5]/text()')
                for h, a in zip(href, area):
                    a = re.search('\S+', a).group()
                    in_url = 'https://cd.daoxila.com' + h
                    self.shop_info(in_url, a)




    def shop_info(self, in_url, area):
        item = {}
        response = requests.get(in_url, headers=self.headers)
        page = etree.HTML(response.text)
        item['area'] = area
        item['hall_num'] = page.xpath('/html/body/div[1]/div[2]/dl/dd[1]/text()')[0]
        item['shop_type'] = page.xpath('/html/body/div[1]/div[1]/ul/li[2]/a/text()')[0]
        item['store_name'] = page.xpath('/html/body/div[1]/div[2]/h1/text()')[0]
        item['price'] = page.xpath('/html/body/div[1]/div[2]/dl/dd[4]/text()')[0]
        item['address'] = page.xpath('/html/body/div[1]/div[2]/div[1]/text()')[0]
        item['table_max'] = page.xpath('/html/body/div[1]/div[2]/dl/dd[3]/text()')[0]
        self.insert_sql('chengdu_hall', item)

    def insert_sql(self, dbname, item):
        key = ''
        value = ''
        for k, v in item.items():
            key += k + ','
            v = str(v).replace('"', "'")
            value += '"' + str(v) + '"' + ','
        sql = 'insert into %s(%s)value(%s)' % (dbname, key[:-1], value[:-1])
        try:
            self.cursor.execute(sql)
            self.db.commit()
        except Exception as e:
            print(e)


    def insert(self):
        sql = """select id, price,hall_num,table_max from chengdu_hall"""
        self.cursor.execute(sql)
        ress = self.cursor.fetchall()
        for res in ress:
            aa = re.findall('\d+', res[1])
            price = aa[0] + '-' + aa[1] + '/桌'
            hall_num = re.findall('\d+',res[2])[0]
            bb = re.findall('\d+', res[3])
            print(res[3])
            print(bb)
            table_max = bb[1]
            up = """update chengdu_hall set price = '{}', hall_num = '{}', table_max = '{}' where id = '{}'""".format(price,hall_num,table_max,res[0])
            print(up)
            self.cursor.execute(up)
            self.db.commit()




if __name__ == '__main__':
    d = DXL()
    d.insert()

以上就是基本的到喜啦酒店信息抓取过程