request-html爬取 58商铺信息

# -*- coding: utf-8 -*-
from requests_html import HTMLSession
import requests
import time
import random
import pymysql
import demjson

session = HTMLSession()
#生成所有分页链接
#rootUrl = https://sjz.58.com/shangpucz/0/pn+pageNum
def getAllPages(rootUrl,num):
    urlList = []
    for page in range(1,num):
        urlList.append(rootUrl+str(page))
    return urlList

#获取每页商铺列表链接
def getShopLinks(urls):
    urlList = []
    headers = {
        'Host': 'sjz.58.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
        'Cookie': 'f=n; commontopbar_new_city_info=241%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7Csjz; userid360_xml=BA6DEAC7B068D4CA2C0B084D64B2C988; time_create=1557562756785; f=n; commontopbar_new_city_info=241%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7Csjz; id58=c5/njVyUiEkue+APA2AWAg==; 58tj_uuid=0382a3da-caf0-41ed-9711-9a412f14cbcc; new_uv=4; als=0; xxzl_deviceid=zd9%2BGdOPhhmD68uINznQ4Kj6XA%2BaAR%2BMD9jB%2FpK0UvOLwPDosI65BYdMB8V%2BxNWL; mcity=sjz; city=sjz; 58home=sjz; wmda_uuid=ceb37f71d389b50a9f62a743a6407c4a; wmda_new_uuid=1; wmda_visited_projects=%3B2385390625025; ppStore_fingerprint=C05282C938C37C02AA98F51C8C619E78B0D04E47FDE636B0%EF%BC%BF1554971585376; f=n; commontopbar_new_city_info=241%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7Csjz; commontopbar_ipcity=sjz%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7C0; JSESSIONID=C8B96CE799B9BDAD7390E594ED5E119B; new_session=1; utm_source=; spm=; init_refer=',
        'Upgrade-Insecure-Requests': '1',
        'Cache-Control': 'max-age=0',
    }
    for i in urls:
        r = session.get(i)
        htmlContent = r.html
        liContent   = htmlContent.find('.content-side-left li')#获取页面所有商铺列表
        for url in liContent:
            linkStr = url.find('.pic > a')[0].attrs['href']
            urlList.append(linkStr)#获取每条商铺信息的详情链接并插入列表
            print(linkStr)
        time.sleep(3)
    return urlList




#获取商铺详情数据
def getShopDetail(shopLink):
    try:
        dictDetail = {}
        r = session.get(shopLink)
        htmlContent = r.html
        #商铺标题
        title = htmlContent.find('.house-title > h1')[0].text
        #铺主联系电话
        linkmanTel = htmlContent.find('.phone-num')[0].text
        #铺主姓名
        linkman    = htmlContent.find('.agent-name')[0].text
        #月租金
        rent = htmlContent.find('.house_basic_title_money > .house_basic_title_money_num')[0].text
        #配套设施
        peitao = htmlContent.find('.peitao-on')
        peitaoList = []
        for i in peitao:
            peitaoList.append(i.text)
        
        #商铺图片
        picList = htmlContent.find('.general-pic-list > li > img')
        urlList = []
        for i in picList:
            urlList.append(i.attrs['src'])
        #面积 类型 楼层 规格 状态 付款方式 经营行业 起租期  位置
        basicContent1 = htmlContent.find('.house_basic_title_content_item2')
        basicContent2 = htmlContent.find('.house_basic_title_content_item3')
        area        = basicContent1[0].text           #面积
        floor       = basicContent1[1].text           #楼层
        managetype  = basicContent1[2].text           #经营状态
        b_id        = basicContent1[3].text           #经营类型

        shoptype    = basicContent2[0].text           #商铺类型
        miankuan    = basicContent2[1].text           #面宽
        paymethod   = basicContent2[2].text           #付款方式
        startrent   = basicContent2[3].text           #起租期
        keliu       = basicContent2[4].text           #付款方式
        street      = basicContent2[5].text           #街道
        dtailStreet = basicContent2[6].text           #详细街道
        #打印数据
        dictDetail['title'] = title
        dictDetail['linkman'] = linkman
        dictDetail['linkmanTel'] = linkmanTel
        dictDetail['rent'] = rent
        dictDetail['area'] = area
        dictDetail['floor'] = floor
        dictDetail['managetype'] = managetype
        dictDetail['b_id'] = b_id
        dictDetail['shoptype'] = shoptype
        dictDetail['miankuan'] = miankuan
        dictDetail['paymethod'] = paymethod
        dictDetail['startrent'] = startrent
        dictDetail['keliu'] = keliu
        dictDetail['street'] = street
        dictDetail['dtailStreet'] = dtailStreet
        dictDetail['peitaoList'] = demjson.encode(peitaoList)
        dictDetail['urlList'] = demjson.encode(urlList)
        print('商铺标题：' + title)
        print('铺主姓名：' + linkman)
        print('铺主联系电话：' + linkmanTel)
        print('商铺租金：' + rent)
        print('面积：' + area)
        print('楼层：' + floor)
        print('经营状态：' + managetype)
        print('经营类型：' + b_id)

        print('商铺类型：' + shoptype)
        print('面宽：' + miankuan)
        print('付款方式：' + paymethod)
        print('起租期：' + startrent)
        print('客流来源：' + keliu)
        print('街道：' + street)
        print('详细街道：' + dtailStreet)
        print('配套设施:')
        print(peitaoList)
        print('商铺图片:')
        print(urlList)
        return dictDetail
    except:
        print('详情出错')

# 将数据生成txt文件方法 传入保存文件路径 storagePath 以及文件数据 data
def storageToLocalFiles(storagePath, pageList):
    content = ''
    fhandle = open(storagePath,"w")
    for i in pageList:
        content += i + '\n'
    fhandle.write(content)
    fhandle.close()


#程序执行
if __name__ == "__main__":

    #获取所有列表页面
    pageList = []
    rootUrl = 'https://sjz.58.com/shangpucz/0/pn'
    pageList = getAllPages(rootUrl,71)
    #print(pageList)

    # # 所有页面链接存储到txt文档
    storagePath = "shopList.txt"
    storageToLocalFiles(storagePath, pageList)

    # #所有需要爬取的商铺详情链接
    shopList = []
    shopList = getShopLinks(pageList)
    # #商铺详情链接储存到txt文档
    storagePath = "shopDetailList.txt" #定义储存文件名
    storageToLocalFiles(storagePath, shopList)

    # f = open('surl.txt','r')
    # string = f.read()
    # shopList = string.split("\n")
    # # print(shopList)
    # f.close()

    #商铺详情字段
    for url in shopList:
        try:
            t = random.randint(2,4)
            dictDetail = getShopDetail(url)
            #插入数据库
            db = pymysql.connect('127.0.0.1', port = 3306, user = 'root', passwd = 'root', db = 'wbug')
            cursor = db.cursor()
            sql = "insert into shop(title,linkman,linkmantel,rent,area,floor,managetype,b_id,shoptype,miankuan,paymethod,startrent,keliu,street,dtailstreet,peitaolist,urllist) values ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}')".format(dictDetail['title'],dictDetail['linkman'],dictDetail['linkmanTel'],dictDetail['rent'],dictDetail['area'],dictDetail['floor'],dictDetail['managetype'],dictDetail['b_id'],dictDetail['shoptype'],dictDetail['miankuan'],dictDetail['paymethod'],dictDetail['startrent'],dictDetail['keliu'],dictDetail['street'],dictDetail['dtailStreet'],dictDetail['peitaoList'],dictDetail['urlList'])
            re = cursor.execute(sql)
            time.sleep(t)
        except:
            print('插入数据库出错')
    db.close()
    cursor.close()

欢迎交流评论建议~