首先说一下这是我自己封装的一个puppteer的浏览器实例池,代码如下
import puppeteer from "puppeteer";
import {EventEmitter} from 'events';
import {isImageOrVideo} from "../utils/utils.js";
import {config as browserPoolConfig} from "../config/spider.js";
const {headless, poolSize} = browserPoolConfig;
EventEmitter.setMaxListeners(0);
const queue = [];
for (let i = 0; i < poolSize; i++) {
const browser = await puppeteer.launch({headless});
queue.push(browser);
}
/**
* 获取一个新的浏览器实例
* @returns {Promise<Browser|*>}
*/
const getBrowser = async () => {
if (queue.length === 0) {
return await puppeteer.launch({headless});
}
let res = queue.shift();
return res;
};
/**
* 获取浏览器的网页并阻止该网页内的图片和视频请求这样可以防止超时问题
* @param browser
* @returns {Promise<*>}
*/
const getBrowserPageWithoutPicAndVideo = async (browser) => {
// 获取page
let page = await b