今天爬了今日头条,今日头条反爬真强,就1次就给我IP封了12小时,下次得用代理IP!!!
废话不多说,直接甩代码:
import requests
import re
from time import sleep
from urllib.parse import urlencode
from urllib.request import urlretrieve
import json
data={
'offset':0, #这里可以通过offset的值来获取下一页
'format':'json',
'keyword':'街拍',
'autoload':'true',
'count':'20',
'cur_tab':'1',
'from':'search_tab', #这里也可以是 gallery,就是突击里的图片了
}
url='https://www.toutiao.com/search_content/?'+urlencode(data) #编码形成url 注意看浏览器上的头条网址的&符号
header={'User-Agent':'User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}
web_urls=list()
def gethtml(url):
html=requests.get(url)
if html.status_code==200:
pass
else:
print('未成功')
return html.json() #转成json格式
def get_article_urls(url): #获取每一个文章的具体url
js=gethtml(url)
group_list=js.get('data')
for item in group_list:
web_urls.append(item.get('article_url'))
def down(web_urls): # 下载函数
j=0
http_list=[]
for item in web_urls:
if item != None:
html=requests.get(item,headers=header).text
pattern=r'"http:\\\\(.*?)\"'
pattern=re.compile(pattern)
real_urls=re.findall(pattern,html)
for tem in real_urls:
st=''
for i in range(len(tem)):
if tem[i]=='/' or tem[i]=='.' or tem[i]=='-' or (tem[i]>='0' and tem[i]<='9') or (tem[i]>='a' and tem[i]<='z'):
st=st+tem[i]
st='http:'+st
http_list.append(st)
for i in range(0,len(http_list),4):
urlretrieve(http_list[i],r'C:\Users\镇长\Desktop\爬虫测试照片\test{0}.png'.format(j))
print('OK')
j=j+1
sleep(2) #预防被封IP
#开始调用函数
get_article_urls(url)
down(web_urls)