java爬虫监控,爬虫利器 Puppeteer实战全自动，监控登陆后页面的数据

const puppeteer = require('puppeteer');

// 等待3000毫秒

const sleep = time => new Promise(resolve => {

setTimeout(resolve, time);

})

//const url = `https://movie.douban.com/explore#!type=movie&tag=%E7%BB%8F%E5%85%B8&sort=rank&page_limit=20&page_start=0`;

const url = `http://www.xiaoyaoji.cn/dashboard`;

;(async() => {

console.log('Start visit');

// 启动一个浏览器

const brower = await puppeteer.launch({

args: ['--no-sandbox'],

//dumpio: false,

headless: true

});

const page = await brower.newPage() // 开启一个新页面

// 去豆瓣那个页面

await page.goto(url, {

waitUntil: 'networkidle2' // 网络空闲说明已加载完毕

});

//await sleep(5000);

// 点击搜索框拟人输入 鬼才会想起

const name = 'mayanjun@haowumc.com';

await page.type('.text.pristine.untouched', name, {delay: 0});

const pwd = '密码';

await page.type('.text.pristine.untouched', pwd, {delay: 1});

const inputElement = await page.$('input[type=submit]');

await inputElement.click();

await page.waitForNavigation();

console.log(page.url());

await page.goto(url, {

waitUntil: 'networkidle2' // 网络空闲说明已加载完毕

});

// 结果

const BRANDS_INFO_SELECTOR = 'div.mc.home-projects';

const result = await page.evaluate(sel => {

// 拿到页面上的jQuery

const ulList = Array.from($(sel).find('div.col-sm-2'));

const ctn = ulList.map(v => {

const title = $(v).html();

return {

title: title,

};

});

return ctn;

},BRANDS_INFO_SELECTOR);

brower.close();

console.log(result);

console.log('End visit');

// 关闭浏览器

})();

转载至链接:https://my.oschina.net/Alexmyj/blog/1919934