使用selenium库的webdriver调用Chrome浏览器访问频道页面并存入文件的代码如下:
from selenium import webdriver
import time
form urllib import parse
import os
baseurl = 'http://xxx.xxxx.xxx/xxx.do'
# query文件的前缀,每个query文件都有多条channel的属性信息
query_fileadd = r'D:\xxx\query'
# 每个query文件对应一个装channelpage的文件夹
channelpage_fileadd = r'D:\xxx\ChannelPages\Query'
start, end = 1, 10
# 初始化一个Chrome的driver
browser = webdriver.Chrome()
for i in range(start, end+1):
# 读取第i个query文件的所有queries
query_filename = query_fileadd + str(i) + '.txt'
qfile = open(query_filename, 'r')
query_list = []
for line in qfile:
query_list.append(line)
qfile.close()
# 确保创建了channelpage的路径
cpdir = os.path.join(channelpage_fileadd, str(i))
if not os.path.exists(cpdir):
os.mkdir(cpdir)
# 根据query_list发送http请求获取channel page
for i in range(len(query_list))
# url的拼接
q = query_list[i].split(' ')
chid, st, et = q[0], q[1], q[2]
pt = str(int(time.time()))
params = dict(startTime=st, endTime=et, channelId=chid, pt=pt)
url = baseurl + '?' + parse.urlencode(query=params)
# 获取channel page的数据
browser.get(url)
channelpagedata = browser.page_source
# 写入channel文件
cpfname = os.path.join(cpdir, 'channel'+str(i)+'.txt')
with open(cpfname, 'w', encoding='utf-8') as cf:
cf.write(channelpagedata)
browser.quit()
上面的代码实现了一键访问下载频道页面的目标,解放了双手,非常开心!然而过了几分钟之后,我眉头一皱,发现 事情并没有这么简单。
之前提到过,浏览器访问一个频道页面大约要10+秒,导致上面调用浏览器的代码下载非常龟速,那么如何能让下载速度提高呢? 多线程。这里我需要的数据没有下载先后的要求,存数据的文件也是每个频道有独立的文件,所以甚至连锁也用不上。
每个线程负责的任务:这里我有多个query的文件,每个文件有多条channel的属性,每条channel属性对应访问一次channel页面。假设query文件一共50个,每个文件有1000条属性,简单来分配的话,每条线程负责10个query文件共5条线程就行(如果内存容量比较大,那可以更多条,速度会相对更快)。
这里多线程我使用的 threading 库,多线程访问并存储的代码如下:
from selenium import webdriver
from urllib import parse
import os
import threading
def getChannelPages(startpage, endpage, query_fileadd, channelpage_fileadd, baseurl, params):
browser = webdriver.Chrome()
for i in range(startpage, endpage, 100):
# 读取文件的query存入query_list
qfname = query_fileadd + str(i) + '.txt'
qfile = open(qfname)
query_list = []
for line in qfile:
query_list.append(line)
qfile.close()
# 确保创建了存channel page的路径
cpdir = os.path.join(channelpage_fileadd, str(i))
if not os.path.exists(cpdir):
os.mkdir(cpdir)
# 根据query_list发送http请求获取channel page
for index in range(len(query_list)):
# url的拼接
query = query_list[index].split(' ')
chid, st, et = query[0], query[1], query[2]
pt = str(int(time.time()))
params['startTime'] = st
params['endTime'] = et
params['channelId'] = chid
url = baseurl + '?' + parse.urlencode(query=params)
# 获取channel page数据
browser.get(url)
channelpagedata = browser.page_source
# 写入channel文件
cpfadd = os.path.join(cpdir, 'channel' + str(index) + '.txt')
with open(cpfadd, 'w', encoding='utf-8') as cf:
cf.write(channelpagedata)
browser.quit()
if __name__ == '__main__':
query_fileadd = r'D:\xxx\query'
channelpage_fileadd = r'D\xxx\ChannelPages'
# channel页面参数设置: baseurl, params
baseurl = 'http://xxxx.xxxx.xxx/xxx.do'
params = dict(startTime='',endTime='',channelId='')
threads = []
# 初始化各线程
startpage, endpage = 1, 1000
for i in range(startpage, endpage, 200):
t = threading.Thread(target=getChannelPages, args=(i, i+199, query_fileadd, baseurl, params))
threads.append(t)
# 开始表演
for t in threads:
t.start()
for t in threads:
t.join()
目前代码实现了多线程爬虫的功能,大幅提高了爬虫的速度,不过还有很多可以优化的部分(比如换一种线程负责的功能,或者改成多进程,还有异步,除了线程以外,对频道属性的提取可以用更高效的正则表达式而不是find函数)。
完整代码
下面是爬取列表页,提取query,爬取频道页的完整代码,其中爬取列表页也改成多线程方式了。
from selenium import webdriver
from urllib import parse
import time
import re
import os
import threading
def getHistoryPages(startpage, endpage, listpage_fileadd, baseurl, params):
# 访问第startpage页到第endpage页的列表页面,每100页存入一个文件
browser = webdriver.Chrome()
for bigpage in range(startpage, endpage, 100):
if bigpage+99 > endpage:
end = endpage
else:
end = bigpage + 99
page_data = []
for page in range(bigpage, end+1):
params['currentPage'] = str(page)
params['pt'] = str(int(time.time))
url = baseurl + '?' + parse.urlencode(query=params)
browser.get(url)
page_data.append(browser.page_source)
listpage_filename = listpage_fileadd + str(bigpage) + '-' + str(bigpage+99) + '.txt'
lfile = open(listpage_filename, 'w', encoding='utf-8')
for item in page_data:
lfile.write(item+'\n')
lfile.close()
del page_data
browser.quit()
def extractQueries(listpage_filename, query_filename):
# 把给定文件里的page data转换成query
query_str = ['channelId startTime endTime']
file = open(listpage_filename, 'r', encoding='utf-8')
data = file.read()
file.close()
query_count = data.count('"allowVideo"')
index1 = data.find('"allowVideo"')
for i in range(query_count - 1):
index2 = data.find('"allowVideo"', index1 + 1)
query_data = data[index1:index2]
if query_data.find('"type":"1"'):
index_chid = query_data.find('channelId')
chid = re.sub(r'\D', '', query_data[index_chid, index_chid+30])
index_st = query_data.find('startTime')
st = query_data[index_st+12, index_st+19]
index_et = query_data.find('endTime')
et = query_data[index_et+10, index_et+19]
str = chid + ' ' + st + ' ' + et
query_str.append(str)
index1 = index2
# 把query_str写入文件
qfile = open(query_filename, 'w', encoding='utf-8')
for item in query_str:
qfile.write(item+'\n')
def getChannelPages(startpage, endpage, query_fileadd, channelpage_fileadd, baseurl, params):
browser = webdriver.Chrome()
for i in range(startpage, endpage, 100):
# 读取文件的query存入query_list
qfname = query_fileadd + str(i) + '-' + str(i+99) '.txt'
qfile = open(qfname)
query_list = []
for line in qfile:
query_list.append(line)
qfile.close()
# 确保创建了存channel page的路径
cpdir = os.path.join(channelpage_fileadd, str(i) + '-' + str(i+99))
if not os.path.exists(cpdir):
os.mkdir(cpdir)
# 根据query_list发送http请求获取channel page
for index in range(len(query_list)):
# url的拼接
query = query_list[index].split(' ')
chid, st, et = query[0], query[1], query[2]
pt = str(int(time.time()))
params['startTime'] = st
params['endTime'] = et
params['channelId'] = chid
url = baseurl + '?' + parse.urlencode(query=params)
# 获取channel page数据
browser.get(url)
channelpagedata = browser.page_source
# 写入channel文件
cpfadd = os.path.join(cpdir, 'channel' + str(index) + '.txt')
with open(cpfadd, 'w', encoding='utf-8') as cf:
cf.write(channelpagedata)
browser.quit()
if __name__ == '__main__':
listpage_fileadd = r'D:\xxx\xxx\page'
query_fileadd = r'D:\xxx\query'
channelpage_fileadd = r'D\xxx\ChannelPages'
# 列表页面参数设置: baseurl, params
baseurl = 'http://xxxx.xxxx.xxx/xxx.do'
params = dict(startTime='2018-01-01 00:00:00', endTime='2018-01-01 00:00:00')
startpage, endpage = 1, 1000
threads = []
for page in range(startpage, endpage, 200):
t = threading.Thread(target=getHistoryPages, args=(page, page+199, listpage_fileadd, baseurl, params))
threads.append(t)
for t in threads:
t.start()
for t in threads:
t.join()
# 从channel list文件中提取channel的属性
startpage, endpage = 1, 1000
for listpage in range(startpage, endpage, 100):
listpage_filename = listpage_fileadd + str(listpage) + '-' + str(listpage+99) + '.txt'
query_filename = query_fileadd + str(listpage) + '-' + str(listpage+99) + '.txt'
extractQueries(listpage_filename, query_filename)
# channel页面参数设置: baseurl, params
baseurl = 'http://xxxx.xxxx.xxx/xxx.do'
params = dict(startTime='',endTime='',channelId='')
threads = []
startpage, endpage = 1, 1000
for i in range(startpage, endpage, 200):
t = threading.Thread(target=getChannelPages, args=(i, i+199, query_fileadd, baseurl, params))
threads.append(t)
for t in threads:
t.start()
for t in threads:
t.join()
后续
爬下来这么多数据,肯定不能存在同一个电脑里,因此之后还经历了设置ftp server等步骤对爬下来的数据进行转移,这里不细讲了,网上有很多教程。