爬取某人的CSDN按照输入的页码开始爬取并命名写入文件夹
具体代码如下:
import urllib.request
import urllib
import os # 新建文件需要的包
csdn = 'csdn'
start_pag = input('输入开始页数:')
def get_Csdn():
url = 'https://blog.csdn.net/Luzaofa/article/list//' # 爬取数据的目的地址
if not os.path.exists(csdn): # 判断文件夹是否存在
os.mkdir(csdn)
data = str(start_pag)
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',
'accept-language': 'zh-CN,zh;q=0.8',
'content-type': 'text/plain;charset=UTF-8',
'cookie': 'uuid_tt_dd=10_30828741730-1560594991036-762001; dc_session_id=10_1560594991036.417027; firstDie=1; BAIDU_SSP_lcr=https://www.baidu.com/link?url=TiCuydAmUCh0WgQ9HoSJ9gT4ZdyOAtewnlROPdWGHkLkIm_ROtFU1Sfun_8F2LUS&wd=&eqid=bdb864ea002e6878000000035d04df79; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1560594996,1560595188,1560596892,1560600447; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1560600482; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_30828741730-1560594991036-762001; c-login-auto=9; dc_tos=pt538t',
}
url1 = url + data + '?' # 地址连接
request = urllib.request.Request(url=url1, headers=headers) # 发起请求
response = urllib.request.urlopen(request) # 响应请求
def Make_dic(response):
file_name = csdn + '_' + str(start_pag) + '.html' # 新建文件名
file_path = csdn + '/' + file_name # 文件路径
with open(file_path, 'wb') as fp: # 把爬取到的内容写入响应文件
fp.write(response.read())
if __name__ == "__main__":
get_Csdn()
Make_dic()