功能
1.自动登录csdn
2.查找未评论的资源并自动评论
用到的库
1.python自带的requests,获取以及发送网页数据
2.python自带的time,用作休眠,csdn资源一段时间内只允许评论一个资源,每评论一个资源要休息一段时间
3.BeautifulSoup,解析html文本,搜索文本中指定的标签和属性
过程
1.用chrome的开发者工具抓取登录以及评论的数据包,获取数据包格式
2.requests获取指定页面html文本
3.BeautifulSoup分析页面属性,提取评论以及登录必须的页面信息
4.对BeautifulSoup的结果进行筛选,去掉一些干扰的标签信息,并判断资源是否评论过
5.组装评论消息并post。
脚本
import requests
from BeautifulSoup import BeautifulSoup
import time
def commitfunc(source_id, refer):
commiturl= ' http://download.csdn.net/index.php/comment/post_comment?jsonpcallback=jsonp1419934439524&sourceid= '+source_id+ ' &content=%E6%88%90%E5%8A%9F%E9%85%8D%E5%AF%B9%EF%BC%8C%E5%8F%AF%E4%BB%A5%E4%BD%BF%E7%94%A8%E3%80%82&rating=5&t=1419935091974 '
commitdata = {
" Accept " : " text/javascript, application/javascript, */* ",
" Accept-Encoding " : " gzip, deflate, sdch ",
" Accept-Language " : " zh-CN,zh;q=0.8 ",
" Connection " : " keep-alive ",
" Content-Type " : " application/x-www-form-urlencoded ",
" cookie " : "",
" Host " : " download.csdn.net ",
" Refer " : refer,
" User-Agent " : " Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 ",
" X-Requested-With " : " XMLHttpRequest ",
" jsonpcallback " : " jsonp1419934439524 ",
" sourceid " : source_id,
" content " : " a good resource, it's worth to download it ",
" rating " : " 5 ",
" t " : " 1419935091974 ",
}
sess_source.post(commiturl, commitdata)
def GetPageCount():
url_source_page = url_source + " 1 "
html_source = sess_source.get(url_source_page).text
soup_source = BeautifulSoup(html_source)
# resource_once = soup_source.findAll('h3')
page_count = soup_source.find( ' div ', attrs={ ' class ': " page_nav "}).text
page_list = page_count.split()
page_ac = page_list[2].split( ' & ')
return page_ac[0][1:len(page_ac[0])-1]
def CommitWholePage(page_nu):
url_source_page = url_source + page_nu
html_source = sess_source.get(url_source_page).text
soup_source = BeautifulSoup(html_source)
resource_once = soup_source.findAll( ' h3 ')
for element in resource_once:
if(len(element) > 1):
# print type(element.contents[0].attrs[0][1])
# print element.contents[0].attrs[0][1]
attr = element.contents[0].attrs[0][1].split( ' / ')
reftext = ' /detail/ ' + attr[2] + ' / ' + attr[3] + ' #comment '
result = soup_source.findAll( ' a ', attrs={ ' href ' : reftext, ' class ' : ' btn-comment '})
if len(result) != 0:
# sess_source.get(url_source_page)
commitfunc(attr[3], attr[2])
print attr[2]
print attr[3]
print " sleep "
time.sleep(70)
def logincsdn():
html_login = sess_source.get(url_login).text
soup_login = BeautifulSoup(html_login)
lt_value = soup_login.findAll( ' input ', attrs={ ' name ': " lt "})[0][ ' value ']
execution_value = soup_login.findAll( ' input ', attrs={ ' name ': " execution "})[0][ ' value ']
data_login = {
" lt " : lt_value,
" execution " : execution_value,
" _eventId " : " submit ",
" username " : " xxxxx ",
" password " : " xxxxx "
}
sess_source.post(url_login, data_login)
# main begin
url_login = " https://passport.csdn.net/account/login "
url_source = " http://download.csdn.net/my/downloads/ "
sess_source = requests.session()
logincsdn()
total_page = GetPageCount()
for num in range(1,int(total_page)):
CommitWholePage(str(num))
from BeautifulSoup import BeautifulSoup
import time
def commitfunc(source_id, refer):
commiturl= ' http://download.csdn.net/index.php/comment/post_comment?jsonpcallback=jsonp1419934439524&sourceid= '+source_id+ ' &content=%E6%88%90%E5%8A%9F%E9%85%8D%E5%AF%B9%EF%BC%8C%E5%8F%AF%E4%BB%A5%E4%BD%BF%E7%94%A8%E3%80%82&rating=5&t=1419935091974 '
commitdata = {
" Accept " : " text/javascript, application/javascript, */* ",
" Accept-Encoding " : " gzip, deflate, sdch ",
" Accept-Language " : " zh-CN,zh;q=0.8 ",
" Connection " : " keep-alive ",
" Content-Type " : " application/x-www-form-urlencoded ",
" cookie " : "",
" Host " : " download.csdn.net ",
" Refer " : refer,
" User-Agent " : " Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 ",
" X-Requested-With " : " XMLHttpRequest ",
" jsonpcallback " : " jsonp1419934439524 ",
" sourceid " : source_id,
" content " : " a good resource, it's worth to download it ",
" rating " : " 5 ",
" t " : " 1419935091974 ",
}
sess_source.post(commiturl, commitdata)
def GetPageCount():
url_source_page = url_source + " 1 "
html_source = sess_source.get(url_source_page).text
soup_source = BeautifulSoup(html_source)
# resource_once = soup_source.findAll('h3')
page_count = soup_source.find( ' div ', attrs={ ' class ': " page_nav "}).text
page_list = page_count.split()
page_ac = page_list[2].split( ' & ')
return page_ac[0][1:len(page_ac[0])-1]
def CommitWholePage(page_nu):
url_source_page = url_source + page_nu
html_source = sess_source.get(url_source_page).text
soup_source = BeautifulSoup(html_source)
resource_once = soup_source.findAll( ' h3 ')
for element in resource_once:
if(len(element) > 1):
# print type(element.contents[0].attrs[0][1])
# print element.contents[0].attrs[0][1]
attr = element.contents[0].attrs[0][1].split( ' / ')
reftext = ' /detail/ ' + attr[2] + ' / ' + attr[3] + ' #comment '
result = soup_source.findAll( ' a ', attrs={ ' href ' : reftext, ' class ' : ' btn-comment '})
if len(result) != 0:
# sess_source.get(url_source_page)
commitfunc(attr[3], attr[2])
print attr[2]
print attr[3]
print " sleep "
time.sleep(70)
def logincsdn():
html_login = sess_source.get(url_login).text
soup_login = BeautifulSoup(html_login)
lt_value = soup_login.findAll( ' input ', attrs={ ' name ': " lt "})[0][ ' value ']
execution_value = soup_login.findAll( ' input ', attrs={ ' name ': " execution "})[0][ ' value ']
data_login = {
" lt " : lt_value,
" execution " : execution_value,
" _eventId " : " submit ",
" username " : " xxxxx ",
" password " : " xxxxx "
}
sess_source.post(url_login, data_login)
# main begin
url_login = " https://passport.csdn.net/account/login "
url_source = " http://download.csdn.net/my/downloads/ "
sess_source = requests.session()
logincsdn()
total_page = GetPageCount()
for num in range(1,int(total_page)):
CommitWholePage(str(num))
欢迎咨询和指正,python菜鸟一枚。
BeautifulSoup中文资料站:http://www.crummy.com/software/BeautifulSoup/bs3/documentation.zh.html#Iterating%20over%20a%20Tag