spider with puppeteer

学习puppeteer 记录示例

直接上代码：

// spider.js
// 爬取美女网站的图片
const puppeteer = require('puppeteer');
const fs = require("fs");
const rp = require("request-promise");

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto("http://www.haommw.com/");
  const RANK = ".grid-item .img-wrap";
  await page.waitFor(RANK);
  const res = await page.evaluate(() => {
    const getSrc = (v, selector) => {
      return v.querySelector(selector) && v.querySelector(selector).src;
    };
    const salesRank = Array.from(
      document.querySelectorAll(".grid-item .img-wrap")
    );
    const data = [];
    salesRank.map(v => {
      const obj = {
        src: getSrc(v, 'img')
      };
      data.push(obj);
    });
    return data;
  });

  res.map(async (item) => {
    let { src } = item;
    const name = src.substr(src.lastIndexOf('/'), src.length);

    let fetchResult = await rp({url: src, encoding: null});
    fs.writeFileSync(`./download/${name}`, fetchResult);
  });
  await browser.close();
})();

运行方法：

$ npm install
$ node spider.js

至此，结束。