怎么用python爬取老师_requests简单爬取佩奇老师Python目录下的文章内容

#coding:utf-8

import requests

import xlrd

from xlutils.copy import copy

import time

from bs4 import BeautifulSoup

#定义header头(简单)

headers = {

"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"

}

#起始URL

start_url = "http://www.cnblogs.com/wupeiqi/articles/4938499.html"

def run(start_url):

# 发送get请求

html = requests.get(url=start_url,headers=headers,timeout=30)

# 解析成dom树

html = BeautifulSoup(html.text,'html.parser')

#提取出所有文章链接

urls = []

for item in html.select("#cnblogs_post_body a"):

urls.append(item.attrs.get("href"))

#循环链接列表继续发送URL请求文章详细页

print("共找到—— %s篇 ——文章"%len(urls))

count = 1

for url in urls:

print("-------正在下载文章 %s/%s--------"%(count,len(urls)))

count+=1

article = requests.get(url=url,headers=headers,timeout=30)

process_article(article,url)

print("下载完成 谢谢使用！".center(80,"-"))

def save_article(title,content,url):

#将数据存到表格

workbook = xlrd.open_workbook('article.xls') #打开表格

nrows = int(workbook.sheets()[0].nrows) #获取表格现有的行数

workbooknew = copy(workbook)

ws = workbooknew.get_sheet(0) #获取第一张工作表

ws.write(nrows,0,str(title)) #写入标题

ws.write(nrows,1,str(url)) #这里只保存了标题和链接，内容太长表格里放不下

workbooknew.save('article.xls') #更新表格

def process_article(article,url):

# 解析成dom树

article = BeautifulSoup(article.text,'html.parser')

#提取出文章的标题和内容

title = article.select("#cb_post_title_url")[0].text

print("正在下载文章----->> %s"%title)

content = article.select("#topics .blogpost-body")

#.children

save_article(title,content,url)

if __name__ == "__main__":

run(start_url)