车质网爬虫

import requests,re,json,jsonpath,time
from bs4 import BeautifulSoup
import pandas as pd

#投诉编号 投诉品牌 投诉车系 投诉车型 问题简述 典型问题 投诉时间 投诉状态
result=pd.DataFrame(columns=['id','brand','car_modle','type','desc','problem','datatime','status'])

def get_info(request_url):
    headers={
   'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'}
    html=requests.get(request_url,headers=headers,timeout=10)
    content=html.text
    soup=BeautifulSoup(content,'html.parser')
    temp=soup.find('div',class_='tslb_b')
    #temp里面所有的tr就是汽车投诉那个表
    tr_list=temp.find_all('tr')
    ##投诉编号 投诉品牌 投诉车系 投诉车型 问题简述 典型问题 投诉时间 投诉状态
    df_info=pd.DataFrame(columns=['id','brand','car_modle','type','desc','problem','datatime','status'])
    for tr in tr_list:#再去获取里面的每一行  每一行的8个<td>就是要拿到的内容
        tempMy={
   }#先用个字典 准备存放每一个td里面的内容
        #第一个tr没有td 因为它就是网页中的列索引名称  从第二个tr开始 里面才td 存放的是表格里的内容
        td_list=tr.find_all('td')
        if len(td_list)>0:#如果有td 就去提取里面的内容

            # 各个表格内容分别解构赋值到对应字典
            tempMy['id'],tempMy['brand'],tempMy['car_modle'],tempMy['type'],tempMy['desc'],tempMy['problem'],tempMy['datatime'],tempMy['status']=td_list[0].text,td_list[1].text,td_list[2].text,td_list[3].text,td_list[4].text,td_list[5].text,td_list[6].text,td_list[7].text
            
            #把故障代码替换成明文
            with open('./ctypeinfo.json','r',encoding='utf-8') as f:
                jsondic=json.load(f)
            #故障码正则为字母，数字元组
            codetuple=re.findall(r'(\w)(\d{1,4})',tempMy['problem']) 

            probs=[] #多个故障码则放入此列表
            for code in codetuple: #循环处理每个故障码为明文：字母明文:数字码明文
                abc=code[0] #取出故障码内的字母
                num=code[1] #取出故障码内的数字
                # print(abc,num)
                # 从json文件中解析出故障码字母部分对应的明文
                tar=jsonpath.jsonpath(jsondic,'$[?(@.value=="'+abc+'")].name')[0]
                # print(tar)
                #从json文件中解析出故障码数字部分对应的明文
                tar2=jsonpath.jsonpath(jsondic,'$[?(@.value=="'+abc+'")].items[?(@.id=="'+num+'")].title')[0]
                probs.append(tar+':'+tar2)

            tempMy['problem']=probs #把故障明文列表赋值给
            print(tempMy['problem'])
            #添加到外面那个大的dataframe里面
            df_info=df_info.append(tempMy,ignore_index=True)
    return df_info

url='http://www.12365auto.com/zlts/81-1940-0-0-0-0_0-0-0-0-0-0-0-' #rx5投诉表
for i in range(1,39):
    result=result.append( get_info(url+str(i)+'.shtml') ) 
    time.sleep(1)
result.to_excel("车质网汽车投诉.xlsx",engine='openpyxl',encoding='utf-8-sig',index=False)

json文件可在此网址抓取，记得把id改成字串形式，即加双引号

http://www.12365auto.com/zlts/0-0-0-0-0-0_0-0-0-0-0-0-0-1.shtml
http://www.12365auto.com/js/cTypeInfo.js?version=20211130

var cTypeInfo = [{
   "id":1,"name":"发动机","value":"A","zf":"z","items":[{
   "id":9,"title":"异响"},{
   "id":11,"title":"电子油门延迟"},{
   "id":12,"title":"漏油"},{
   "id":13,"title":"机油乳化"},{
   "id":14,"title":"熄火"},{
   "id":18,"title":"爆震"},{
   "id":24,"title":"抖动"},{
   "id":34,"title":"故障灯亮"},{
   "id":35,"title":"怠速不稳"},{
   "id":37,"title":"漏防冻液"},{
   "id":38,"title":"噪音大"},{
   "id":44,"title":"无法启动"},{
   "id":45,"title":"正时链条及齿轮故障"},{
   "id":53,"title":"喷油嘴故障"},{
   "id":63,"title":"烧机油"},{
   "id":85,"title":"油耗高"},{
   "id":91,"title":"点火线圈故障"},{
   "id":114,"title":"排气故障"},{
   "id":128,"title":"连杆断裂"},{
   "id":129,"title":"曲轴故障"},{
   "id":136,"title":"缸体破损"},{
   "id":137,"title":"喷气嘴故障"},{
   "id":145,"title":"固定螺栓断裂"},{
   "id":160,"title":"无法提速"},{
   "id":163,"title":"功率不足"},{
   "id":173,"title":"火花塞故障"},{
   "id":174,"title":"油门故障"},{
   "id":178,"title":"漏气"},{
   "id":189,"title":"涡轮增压器故障"