python爬虫代码新闻_Python小白的爬虫代码——澎湃新闻列表

在参照豆瓣影评的基础上写的抓取澎湃新闻列表代码

# coding = utf-8

import requests

from lxml import etree

import time

import random

import csv

import codecs

data = []

def getPP(page):

url = 'https://www.thepaper.cn/load_chosen.jsp?nodeids=25949&topCids=2123127,2122823,2120622,2123384,&pageidx=%d'%(page)

response = requests.get(url)

response.encoding = 'utf-8'

response = etree.HTML(response.content,parser=etree.HTMLParser(encoding='utf-8'))#添加默认编码

for i in range(1,21):

title1 = response.xpath('//body/div[%d]/h2/a'%(i))#获取title

url1 = response.xpath('//body/div[%d]/h2/a'%(i))#获取url

summary1 = response.xpath('//body/div[%d]/p'%(i))#获取summary

tag1 = response.xpath('//body/div[%d]/div[2]/a'%(i))#获取文章分类

time1 = response.xpath('//body/div[%d]/div[2]/span[1]'%(i))#获取发布时间

rtimes1 = response.xpath('//body/div[%d]/div[2]/span[2]'%(i))#获取回复数

title_element = title1[0].text.encode('utf-8')

url_element = url1[0].attrib['href']

url_element = 'https://www.thepaper.cn/' + url_element#完善链接

summary_element = summary1[0].text.encode('utf-8')

summary_element = summary_element.replace(' ','')#去空格

tag_element = tag1[0].text.encode('utf-8')

time_element = time1[0].text.encode('utf-8')

if len(rtimes1) == 0:

rtimes_element = 0

else:

rtimes_element = rtimes1[0].text#有些回复数为空，rtimes[0]不存在，会报错，用if函数避免这个问题

data.append([title_element,url_element,summary_element,tag_element,time_element,rtimes_element])

for i in range(1,3):#抓2页

getPP(i)

time.sleep(random.uniform(3,5))

with open ('getPP.csv','wb') as f:

f.write(codecs.BOM_UTF8)

writer = csv.writer(f)

writer.writerow(['title','url','summary','tag','publishtime','rtimes'])

for k in data:

writer.writerow(k)