获取半年内的投诉信息
结束时间
today = datetime.datetime.now()
today = datetime.datetime.strftime(today, "%Y-%m-%d")
today = datetime.datetime.strptime(today, "%Y-%m-%d")
self.endTime = today - datetime.timedelta(weeks=26)
获取每一页详细连接,并存入csv文件
detailPage = response.xpath('//div[@class="col-xs-12 plr_0 ts_list"]/table[2]//tr/@onclick').extract()
match = re.compile(r'[(](.*)[)]', re.S)
for i in range(len(detailPage)):
pageID = re.findall(match, detailPage[i])[0]
detailPage[i] = domain+pageID
获取下一页连接
domain2 ='https://www.aqsiqauto.com'
if self.next == 0:
nextFile = "nextPage.csv"
nextPage = response.xpath('//li[@class="next"]/a/@href').extract()
if nextPage:
self.pageNum = self.pageNum + 1
nextPage = domain2 + nextPage[0]
yield scrapy.Request(nextPage, callback=self.parse)
else:
error1 = '第'+str(self.pageNum)+'页连接获取失败'
with open("erro.csv",'a',encoding='utf-8',newline='') as f:
writer = csv.writer(f)
writer.writerow([self.pageNum,error1])
else:
with open(writeFile, "a", encoding="utf-8", newline="") as f:
writer = csv.writer(f)
writer.writerow(["详细信息数量",self.total])
writer.writerow(["爬取页数",self.pageNum])
print("完成!")
详细页面信息获取
def parse_detail(self, response):
brand = response.xpath('//div[@class="col-xs-12 TS_content plr_0"]/div/div[4]//span/text()').extract()
if not brand:
brand= ['null']
else:
brand[0] = brand[0].strip()
series = response.xpath('//div[@class="col-xs-12 TS_content plr_0"]/div/div[5]//div/text()').extract()
if not series:
series = ['null']
else:
series[0] = series[0].strip()
details = response.xpath(
'//div[@class="col-xs-12 TS_content plr_0"]/div/div[8]//span/text()').extract()
if not details:
details= ['null']
else:
details[0] = details[0].strip()
problem = response.xpath('//div[@class="col-xs-12 TS_content plr_0"]/div/div[7]//span/text()').extract()
if not problem:
problem= ['null']
else:
problem[0] = problem[0].strip()
try:
time = response.xpath(
'//div[@class="col-xs-12 TS_content plr_0"]/div/div[3]//div/text()').extract()
if not time:
time = ['null']
error = [response.meta["id"] + ' 没有时间参数']
with open("error.csv", "a", encoding='utf-8', newline='') as f:
writer = csv.writer(f)
writer.writerow(error)
else:
time[0] = time[0].strip()
pageTime = datetime.datetime.strptime(time[0], '%Y-%m-%d')
diffrence = pageTime - self.endTime
diffrence = diffrence.days
if diffrence < 0:
self.next = 1
except Exception as e:
with open('error.csv','a',encoding='utf-8',newline='') as f:
writer = csv.writer(f)
writer.writerow([self.pageNum,response.meta['id'],e.args])
self.total = self.total+1
item = QautoItem()
item['brand'] = brand
item['series'] = series
item['details'] = details
item['problem'] = problem
item['time'] = time
yield item