爬虫学习笔记--爬取百度贴吧

由于松爱协会小伙伴的邀请 我把贴吧里的一些诗集整理了一下 

用爬虫爬取下来 

由于是静态的 不需要用到selenuim

就直接贴代码了

#coding=utf-8
import requests
from bs4 import BeautifulSoup
import sys
import time
reload(sys)
sys.setdefaultencoding('utf-8')

link = "https://tieba.baidu.com/p/4877675324"
link2 = "https://tieba.baidu.com/p/4877675324?pn=2"

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'}

r = requests.get(link,headers=headers)
r2 = requests.get(link2,headers=headers)

soup = BeautifulSoup(r.text,"html.parser")
soup2 = BeautifulSoup(r2.text,"html.parser")

content_list = soup.find_all("div",class_ = "d_post_content j_d_post_content ")
content_list2 = soup2.find_all("div",class_ = "d_post_content j_d_post_content ")

for i in range(len(content_list)):
    conent = content_list[i].text.strip()
    print ("诗集"+str(i+1)+":")
    print (conent)

for j in range(len(content_list2)):
    conent2 = content_list2[j].text.strip()
    print ("诗集"+str(i+j+1)+":")
    print (conent2)