Python网络爬虫技术案例教程

爬取哔哩哔哩网站视频信息

代码如下：

import time
import random
import params as params
import requests
from urllib.parse import urlencode
from bs4 import BeautifulSoup
import csv

from requests import Response

base_url = 'https://search.bilibili.com/all?'
headersvalue = {
    'user-agent': 'Moilla/5.0(Windows NT 10.0;win64; x64)AppleWebkit/537.36  (KHTML, like Gecko)  Chrome/83.0.4103.97safari/537.36'
}
proxiesvalue = [
    {'http': 'http://121.232.148.167:9000'},
    {'http': 'http://39.105.28.28:8118'},
    {'http': 'http://113.195.18.133:9999'}
]


def parse_content(content):
    items = []
    soup = BeautifulSoup(content, 'lxml')
    video_list = soup.select('.video-list > li')
    for video in video_list:
        title = video.select('.info a')[0].text.strip()
        video_time = video.select('.so-imgTag_rb')[0].text.strip()