网页爬虫 python-python爬虫——爬取网页的中文

# 爬取网页的中文内容

from urllib import request

from bs4 import BeautifulSoup

import pandas as pds

import xlrd

import docx

#读取excel

def excel(fname):

data=pds.read_excel(fname)

return data

def word(school ,content):

#创建内存中的word文档对象

file=docx.Document()

#写入若干段落

file.add_paragraph(content)

#保存

file.save("E:/大四上/毕业论文/招生简章/"+school+"招生简章.docx")

if __name__ == '__main__':

data=excel("C:/Users/Administrator/Desktop/招生简章.xlsx") #读初始数据

nrow=data.shape[0] #获取数据的行数

for i in range(0,nrow):

school = data.values[i][0]

url =data.values[i][1]

print("正在爬第"+str(i)+"个招生简章")

head = {}

# 使用代理

head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19'

req = request.Request(url, headers = head)

response = request.urlopen(req)

html = response.read()

# 创建request对象

soup = BeautifulSoup(html, 'lxml')

# 找出div中的内容

soup_text = soup.find(attrs={'class':'content'})

# 输出其中的文本

content = soup_text.text

word(school ,content)#写入word文档