一个简单Python爬虫

import urllib.robotparser
import requests

rp = urllib.robotparser.RobotFileParser()
rp.set_url("https://www.douyu.com/robot.txt")//网站名可以改，这里用了百度。
rp.read()
useragent = "Baiduspider"
url = 'https://www.douyu.com/directory/all'//改了上面那里的话，记得这里也要调整。
if rp.can_fetch(useragent, url):
    file = requests.get(url)
    data = file.content
    fb = open("bd-html", "wb")
    fb.write(data)
    fb.close()
    print("抓取成功")
else:
    print("不允许爬取")