爬虫练习--使用正则表达式爬取B站排行榜

 

1.首先打开B站网页

从中可以获取的信息是网站的URL是“https://www.bilibili.com/v/popular/rank/all”

2.打开开发者工具，在Network选项卡下查看Cookie

3.查看网站源代码，编写正则表达式

观察源代码，我们可以写出正则表达式

<li data-id.*?num.*?>(.*?)</div>.*?img.*?href="(.*?)".*?</a>.*?title.*?>(.*?)</a>.*?b-icon play.*?</i>(.*?)</span>.*?b-icon view.*?</i>(.*?)</span>.*?b-icon author.*?</i>(.*?)</span>.*?pts.*?<div>(.*?)</div>.*?</li> 

4.抓取页面

#抓取网页
def get_one_page(url):
	headers = {
		'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
		'Cookie' : "_uuid=50C67A24-10FD-241D-42BC-63060BB3AD4640961infoc; buvid3=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; PVID=1; bsource=search_baidu; fingerprint=86c13fa55e20a8f09e66c6b108be9688; buvid_fp=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; buvid_fp_plain=3BBDF6C0-3345-4F0F-934E-8F76451AC03B13425infoc"
		}
	response = requests.get(url,headers = headers)
	if response.status_code == 200:
		return response.text
	return None

def main():
	url = 'https://www.bilibili.com/v/popular/rank/all'
	html = get_one_page(url)

main()

5.利用正则表达式提取网页信息 

def main():
	url = 'https://www.bilibili.com/v/popular/rank/all'
	html = get_one_page(url)
	
	#查看网站源代码，分析源代码格式，编写正则表达式
	pattern = '<li data-id.*?num.*?>(.*?)</div>.*?img.*?href="(.*?)".*?</a>.*?title.*?>(.*?)</a>.*?b-icon play.*?</i>(.*?)</span>.*?b-icon view.*?</i>(.*?)</span>.*?b-icon author.*?</i>(.*?)</span>.*?pts.*?<div>(.*?)</div>.*?</li>'
	items = re.findall(pattern, html,re.S)

    #将提取的信息写入字典
	for item in items:
		
		d = {}
		d['index'] = item[0]
		d['image'] = item[1]
		d['title'] = item[2].strip()
		d['play'] = item[3].strip()
		d['view'] = item[4].strip()
		d['author'] = item[5].strip()
		d['point'] = item[6].strip()

 6.将整理好的字典写入文件

def write_to_file(content):
    #写入“result1.txt”文件
	with open('result1.txt','a',encoding = 'utf-8') as f:
		print(type(json.dumps(content)))
		f.write(json.dumps(content, ensure_ascii = False) + '\n')

7.结果

 工作台：

result1文件：

大功告成！

前20是我爬猫眼TOP的数据（doge）

8.完整代码

import requests
import re
import json

def get_one_page(url):
	headers = {
		'User-Agent':"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
		'Cookie' : "_uuid=50C67A24-10FD-241D-42BC-63060BB3AD4640961infoc; buvid3=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; PVID=1; bsource=search_baidu; fingerprint=86c13fa55e20a8f09e66c6b108be9688; buvid_fp=B2CBDB74-9521-4C16-9283-36AC6E99CD3234758infoc; buvid_fp_plain=3BBDF6C0-3345-4F0F-934E-8F76451AC03B13425infoc"
		}
	response = requests.get(url,headers = headers)
	if response.status_code == 200:
		return response.text
	return None

def write_to_file(content):
	with open('result1.txt','a',encoding = 'utf-8') as f:
		print(type(json.dumps(content)))
		f.write(json.dumps(content, ensure_ascii = False) + '\n')

def main():
	url = 'https://www.bilibili.com/v/popular/rank/all'
	html = get_one_page(url)
	
	pattern = '<li data-id.*?num.*?>(.*?)</div>.*?img.*?href="(.*?)".*?</a>.*?title.*?>(.*?)</a>.*?b-icon play.*?</i>(.*?)</span>.*?b-icon view.*?</i>(.*?)</span>.*?b-icon author.*?</i>(.*?)</span>.*?pts.*?<div>(.*?)</div>.*?</li>'
	items = re.findall(pattern, html,re.S)
	for item in items:
		
		d = {}
		d['index'] = item[0]
		d['image'] = item[1]
		d['title'] = item[2].strip()
		d['play'] = item[3].strip()
		d['view'] = item[4].strip()
		d['author'] = item[5].strip()
		d['point'] = item[6].strip()
		print(d)
		write_to_file(d)

main()

本文是学习了崔大神的《爬虫》的爬猫眼实例的练习，如果有错误的地方的还请大神们指出！对我是莫大的帮助！