简单的爬虫

var fs=require('fs');
var fse = require('fs-extra');
var request = require('request');
var file="./config.json";
var result=JSON.parse(fs.readFileSync(file));
var path = require('path');

function downloadFile(uri, filename) {
    return new Promise((resolve, reject) => {
        var stream = fs.createWriteStream(filename);
        request(uri).pipe(stream).on('close', () => {
            resolve();
        });
    });
}


async function asyncFun() {
  let baseDir = 'images';
  for (let key in result.material_pack) {
    let pack = result.material_pack[key];
    let pack_id = pack.pack_id;
    let packDir = path.join(baseDir,pack_id.toString());
      for (let k1 in pack.pack_content) {
        let item = pack.pack_content[k1];
        console.log(pack_id, item.face_id);
        let fileName = pack_id + '-' + item.face_id +'-big.png';
        let thumbName =pack_id +'-'+item.face_id + '-thumb.jpg';
        let url= 'http://xxxx/'+pack_id+'/big/'+ fileName;
        let thumbUrl ='http://xxx/'+thumbName;
        let originDir = path.join(packDir, 'origin');
        let thumbDir = path.join(packDir, 'thumb');
        let bigFileName = path.join(originDir, fileName);
        let thumbFileName = path.join(thumbDir,thumbName);
        fse.mkdirsSync(path.dirname(bigFileName));
        fse.mkdirsSync(path.dirname(thumbFileName));
        console.log("downloading big: " + url);
        await downloadFile(url, bigFileName);
        console.log("downloading thumb: " + thumbUrl);
        await downloadFile(thumbUrl, thumbFileName);
      }
  }
}

asyncFun()