哔哩哔哩动漫信息获取(只局限于信息收集),存储数据到excel或者mysql
一、确定你所需要数据来源的地址
数据来源页面和地址如下:
数据来源页面
数据来源地址,需要通过浏览器的开发者工具也就是F12 来获取对应的请求地址:
一般来说,像这样返回的json格式的数据最好获取的。。。。。。
二、准备python的依赖包
一般来说用到的插件不会太多,像请求静态页面的请求依赖包:requests、BeautifulSoup,动态js加载页面的:selenium,本文抓取的数据是json格式,所以一般requests依赖包就完成可以使用
所用到的依赖包如下:
requests(请求返回数据)、numpy(创建空数组)
pymysql(数据入库)、xlwt(数据写入文件xls)
三、代码编写
1、请求获取数据
def ts_requests(baseurl):
data = {
"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
}
result = requests.get(url=baseurl,headers=data)
result.encoding = 'UTF-8'
if result.status_code == 200:
result_data = result.json()
return result_data
else:
print("请求失败,状态码为:"+result.status_code)
2、处理响应数据
def ts_data_deal(url_front,url_end,numgegin,numend):
real_data = np.empty((0,10),str)
j = 0
for ppp in range(numgegin, numend):
baseurl = url_front + str(ppp) + url_end
ts_data = ts_requests(baseurl)['data']['list']
for i in ts_data:
for key, value in i.items():
if type(value) == dict:
if key == 'badge_info':
data_badge = value['text'] # 观看需要条件
elif key == 'first_ep':
data_cover = value['cover'] # 图片链接
data_ep_id = value['ep_id'] # id
else:
pass
elif type(value) == int:
pass
else:
pass
data_link = i['link'] # 地址链接
data_order = i['order'] # 追番数量
data_subtitle = i['subTitle'] # 副标题
data_title = i['title'] # 主标题
data_index_show = i['index_show'] # 集数
if i['score'] =='':
data_score = '暂无评分'
else:
data_score = i['score'] + '分' # 评分
j+=1
real_data = np.append(real_data,np.array([[j,data_badge,data_title,data_subtitle,data_cover,data_index_show,data_score,data_order,data_link,data_ep_id]]),axis=0)
return real_data
3、数据入库
def ts_pymysql(datalist):
conn = pymysql.connect(
host='地址',
port='端口',
user='用户',
password='密码',
db='数据库'
)
cur = conn.cursor()
sql = """drop table if exists table_name"""
print('正在删除对应的表以及相关的数据,请稍等5秒钟.......')
time.sleep(2)
cur.execute(sql)
conn.commit()
print('删除成功!!!')
sql_create = """CREATE TABLE table_name (
id INT PRIMARY KEY NOT NULL,
data_badge VARCHAR ( 255 ),
data_title VARCHAR ( 255 ),
data_subtitle VARCHAR ( 255 ),
data_index_show VARCHAR ( 255 ),
data_order VARCHAR ( 255 ),
data_score VARCHAR ( 255 ),
data_link VARCHAR ( 255 ),
data_cover VARCHAR(225),
data_ep_id VARCHAR(225))
"""
try:
cur.execute(sql_create)
print("table_name创建表成功")
sql_insert = """insert into table_name (id,data_badge,data_title,data_subtitle,data_index_show,data_order,data_score,data_link,data_cover,data_ep_id) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
print('正在导入数据,请稍等5秒钟。。。。')
for item in datalist:
cur.execute(sql_insert,(int(item[0]),item[1],item[2],item[3],item[4],item[5],item[6],item[7],item[8],item[9]))
conn.commit()
print('导入成功,请查看')
except Exception as e:
print(e)
4、存储为表格excel
def tx_excel(datalist):
workBook = xlwt.Workbook('UTF-8')
print('正在检查文件是否存在。。。')
sheet =workBook.add_sheet('bilibili_动漫_数据')
i = 0
print('正在写入操作,请等待一会')
time.sleep(5)
for num in datalist:
for j in range(0,len(num)):
sheet.write(i,j,num[j])
i+=1
workBook.save('./bilibili_动漫_数据.xls')
print('写入完成')
这里的目录或者文件检查等,你可以os模块进行判断
path=“path\\”
if not os.path.isdir(path):
os.mkdir(path)
4、其他汇总
访问:
def main():
#动漫番剧地址
url_front = "https://api.bilibili.com/pgc/season/index/result?st=1&order=3&season_version=-1&spoken_language_type=-1&area=-1&is_finish=-1©right=-1&season_status=-1&season_month=-1&year=-1&style_id=-1&sort=0&page="
url_end = "&season_type=1&pagesize=20&type=1"
#国创地址
#url_front1 = "https://api.bilibili.com/pgc/season/index/result?st=4&order=3&season_version=-1&is_finish=-1©right=-1&season_status=-1&year=-1&style_id=-1&sort=0&page="
# url_end1 = "&season_type=4&pagesize=20&type=1"
num1 = int(input("请输入你要爬取页面数据的开始页:"))
num2 = int(input("请输入你要爬取页面数据的结束页:"))
ppppp = ts_data_deal(url_front, url_end,num1,num2+1)
time.sleep(2)
choose = int(input("请选择存储方式:1、mysql 2、excel 3、pic下载 请输入数字:"))
if choose == 1:
ts_pymysql(ppppp)
elif choose == 2:
tx_excel(ppppp)
elif choose ==3:
tx_picture(ppppp)
else:
exit()
if __name__ == '__main__':
main()