bs4 乱码_爬虫 正则 bs4 xpath 中文乱码 管道符

#爬取糗事百科

# re正则匹配

import requests

import re

import os

headers={

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'

}

dirname="./qiutu"

# 创建文件夹

if not os.path.exists(dirname):

os.mkdir(dirname)

url="https://www.qiushibaike.com/imgrank/page/%d/"

for page in range(1,3):

print("开始下载第{}页图片".format(page))

#指定新的url

new_url=format(url%page)

#获得源码文本信息

page_text=requests.get(url=new_url,headers=headers).text

ex='

.*? '

# 获取图片地址 注意re.S

img_src_list=re.findall(ex,page_text,re.S)

for img_src in img_src_list:

new_img_src='https:'+img_src

img_name=img_src.split("/")[-1]

img_path=dirname+"/"+img_name

img_text=requests.get(url=new_img_src,headers=headers).content

with open(img_path,"wb") as f:

f.write(img_text)

print(img_name,"下载完毕！")

# 方法2:

# from urllib import request

# request.urlretrieve(new_img_src,img_path)

#

#

#

#

#