老司机开车记

最新推荐文章于 2024-11-18 12:11:58 发布

_vip

最新推荐文章于 2024-11-18 12:11:58 发布

阅读量794

点赞数 1

分类专栏： JavaScript语言文章标签： javascript node.js source

本文链接：https://blog.csdn.net/michael_jordaner/article/details/51288671

版权

JavaScript语言专栏收录该内容

3 篇文章 0 订阅

订阅专栏

node.js+jsdom 小爬虫并不是框架>.<

解决某些个人问题:

利用了dfs搜索

"use strict";
let https=require('https');
let fs=require('fs');
let path=require('path');
let jsdom=require('jsdom');
let visted=[];


function isDir(url){
  if(url.indexOf('.')!==-1){
    return false;
  }
  try{
    fs.mkdirSync(url);
    console.log('mkdir:'+url);
  }catch(e){
    console.log(`IOerror:${e}`)
  }
  return true;
}

function isUrlVisited(url){
  for(let i=0;i<visted.length;i++){
    if(visted[i]===url){
      return true;
    }
  }
  visted.push(url);
  return false;
}

function writeFile(url){
  if(isDir(url)&&isUrlVisited(url)){
    return;
  }
  let filePath=path.parse(url);
  try{
    urltoFile(url);
  }catch(error){
    console.log(error)
  }

}

function urltoFile(url){
  https.request(root+url,
  (res)=>{
    res.on('data',(data)=>{
      fs.appendFileSync(url,data);
    });
  }).on('error',(e)=>{
    console.log(e);
  }).end();
}


function dealUrl(url){
  isUrlVisited(url);
  isDir(url);
  writeFile(url);

}
let root='https://www.seryox.com';
function applyUrl(url){
  jsdom.env({
    url: url,
    scripts: ["http://code.jquery.com/jquery.js"],
    done: function (err, window) {
        let $=window.$;
        console.log('done');
        try{
        let arr=$('a');
        console.log(arr.length)
        for(let i=0;i<arr.length;i++){
          let href=$(arr[i]).attr('href');
            if(href.match(/^\/pic/)&&!isUrlVisited(href)){
              console.log(href);
              if(isDir(href)){
                console.log('dir:'+url+href);
                applyUrl(root+href);
              }else{
                console.log('file:'+url+href);
                writeFile(href);
              }
            }
        }
      }catch(e){
        console.log(e+'@'+url);
      }

    }
  });
}
applyUrl(root+'/');

生成的文件夹编码为UTF-8 URL编码

利用此文件可解决

"use strict";
let fs=require('fs');
let root='/pic';

function isDir(url){
  if(url.indexOf('.')!==-1){
    return false;
  }
  return true;
}

function rename(path,name){
  if(name.indexOf('%')!==-1)
  console.log(path+'/'+name+' to '+path+'/'+decodeURI(name))
    fs.renameSync(path+'/'+name,path+'/'+decodeURI(name));
}

function main(path){
  let fd=fs.readdirSync(path);
  for(let i=0;i<fd.length;i++){
    if(isDir(fd[i])){
      main(path+'/'+fd[i]);
    }
    rename(path,fd[i]);
  }
}
main(root);