安装requests库
pip install requests
定义IP地址和请求头
url = "https://ssr1.scrape.center/"
headers = {
'User-Agent':'自己电脑的请求头',
}
爬取文本数据
请求数据
resp = requests.get(url, headers=headers)
获取数据文本
data = resp.text
文本写入
def write_text(path,data):
with open(path, mode='w') as f:
f.write(data)
write_text("templates/ssr1.html", data)
爬取图片数据
import re
import requests
from write_and_read import write_pic
from lxml import html
url = '网站ip地址'
headers = {
'User-Agent':'自己电脑的请求头',
}
resp = requests.get(url, headers=headers)
print(resp)
tree = html.fromstring(resp.text)
print(tree)
img_urls = tree.xpath('//img/@src')
print(img_urls)
for num, pic_url in enumerate(img_urls):
try:
pic_resp = requests.get(pic_url, headers=headers)
parts = pic_url.split('//')[1].split('/')
print(parts[-1])
write_pic(f"statics/image/{parts[-1]}", pic_resp.content)
except:
continue