# -*- coding: utf-8 -*
from bs4 import BeautifulSoup
import urllib.request
import urllib.parse
import re
import urllib.error
import urllib.parse
import pandas as pd
'''第一步:爬去整个网页内容'''
url_range = range(1, 11)
vot_range = ('260', '262', '246', '132', '181', '215', '655', '80', '222', '547')
for n, m in zip(url_range, vot_range):
url = "http://fendou.itcast.cn/article/show/vote/" + str(n) #设置要爬去的url
user_agent = "Mozilla"
headers = {"User-Agent": user_agent} #将爬虫隐藏为Mac用户(固定格式)
request = urllib.request.Request(url, headers=headers) #使用Mac 用户身份登录请求
reponse = urllib.request.urlopen(request).read() #获取目的地址文件内容
'''第二步:正则表达式选取所需要原始段落内容'''
reponse = reponse.decode('utf-8')
soup = BeautifulSoup(reponse, "lxml")
a = list(soup.find("title"))
a2 = list(soup.find('em', {'id': 'votenum'+ str(m)}))
a3 = list(soup.find("p", {"class": "wzcon-p"}))
'''
result2 = re.compile("<span>") #"title"可以改为正则表达式
result3 = result2.sub("", str(a3)) #去除不需要的信息
for i in soup.find_all("div", class_="wzcon-cen"):
ad = i.get_text()
for i in soup.find_all("div", class_="wzbot-left"):
aq = i.get_text()
'''
ah = pd.DataFrame(a)
ah.columns = ['文章标题']
ad = pd.DataFrame(a2)
ad.columns = ["点击量"]
az = pd.DataFrame(a3)
ak = pd.DataFrame(az.ix[1, :])
ak.columns = ["作者名字"]
'''
result2 = re.compile("<span>") #"title"可以改为正则表达式
result3 = result2.sub("", str(ak)) #去除不需要的信息
result4 = re.compile("</span>") #"title"可以改为正则表达式
result5 = result4.sub("", str(result3)) #去除不需要的信息
'''
z = ah.join(ad)
k = z.join(ak)
print(k)
# k.to_csv('F:/RStudio/zaofa.csv', index=False)
'''
file = open("./downloads/31.html", "wb") #相对地址格式为( ./ )
file.write(reponse)
file.close() #将文件保存到本地,并且关闭文件
'''