直接上代码
class GitHub:
def __init__(self):
self.host = "https://github.com"
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"}
self.url_queue = Queue()
self.html_queue = Queue()
self.content_list_queue = Queue()
def init_url(self):
self.url_queue.put("https://github.com/pingcap/tidb/issues")
self.url_queue.put("https://github.com/pingcap/tidb/issues?q=is%3Aissue+is%3Aclosed")
self.url_queue.put("https://github.com/pingcap/tidb/pulls")
self.url_queue.put("https://github.com/pingcap/tidb/issues?q=is%3Apr+is%3Aclosed")
def parse_url(self):
while True:
url = self.url_queue.get()
try:
urllib3.disable_warnings()
re