node爬虫

 const  http = require('http');
 const  fs = require('fs');
   const  moment = require('moment');
let  baseUrl = "http://data.cma.cn/weatherGis/web/bmd/VisDataDef/getVisData?datacode=";

let kindArr = [
    {
        name:"qiwen",
        dataCode: "NAFP_CLDAS2.0_RT_JPG_TMP",

    },
    {
        name:"shidu",
        dataCode: "NAFP_CLDAS2.0_RT_JPG_SHU",
    },
    {
        name:"fengsu",
        dataCode: "NAFP_CLDAS2.0_RT_JPG_WIN",
    },
    {
        name:"qiya",
        dataCode: "NAFP_CLDAS2.0_RT_JPG_PRS",
    },
    {
        name:"jiangshui",
        dataCode: "NAFP_CLDAS2.0_RT_JPG_PRE",
    },
    {
        name:"duanbofushe",
        dataCode: "NAFP_CLDAS2.0_RT_JPG_SSRA",
    }
]
let startTime = moment("20170501");
function getData(rq,dataCode,filename){
    let date = moment(rq).format("YYYYMMDD");
    let url = baseUrl + dataCode + "&d_datetime=" + date;
    http.get(url, function(response){
        let data = "";
        response.setEncoding('utf8')
        response.on('data', function(chunk){
            data+=chunk;
        })
        response.on('end', function(){
            data = JSON.parse(data);
            let imgArr = {
                date : data.maxdate,
                data : []
             };
             for(obj of data.data){
                imgArr.data.push(obj.fileURL);
            }
             imgArr = JSON.stringify(imgArr);
             let newBuildPath = "./data/" + filename + "/"+data.maxdate+'.txt';
                /*
                * 今天之前的数据
                * */
             if(moment(rq).add(1,"hour").isBefore(moment())){
                fs.access(newBuildPath,fs.constants.F_OK,(err)=>{
                       /*
                       * 文件不存在的情况下
                       * */
                     if(err){
                          // console.log("今天之前的数据");
                        fs.appendFile(newBuildPath,imgArr,()=> {;
                            if (startTime.isBefore(date)) {
                                rq = moment(date).subtract(1, 'day').format("YYYYMMDD");
                                getData(rq, dataCode, filename);
                             }
                         })
                     }else{
                        console.log("今天前的文件已经存在了");
                     }
                 });
                    /*
                    * 今天的数据
                    * */
             }else{
                 console.log("今天的文件");
                 fs.writeFile(newBuildPath,imgArr,()=> {;
                     if (startTime.isBefore(date)) {
                         rq = moment(date).subtract(1, 'day').format("YYYYMMDD");
                         getData(rq, dataCode, filename);
                     }
                 })
             }
         })
     }).on('error', function(e){
         console.log(e.message);
     })
 }
 kindArr.forEach( (e,i) => {
     getData(moment(),e.dataCode,e.name);
 })