Koa2 + Puppeteer打造『爬虫系统』2

四.将图片资源上传至七牛图床

1.安装nanoid qiniu

npm i nanoid qiniu  -S

2.建立config文件夹，建立config.js

module.exports={
    qiniu:{
        keys:{
            ak:'',
            sk:''
        },
        buket:{
            tximg:{
                bucket_name:'cwraler-txclass',
                domain:''
            }
        }
    }
}

3.utils修改

const cp = require('child_process'),
      {resolve} = require('path'),
      nanoId = require('nanoid'),
      Qiniu = require('qiniu');
const { info } = require('console');
module.exports={
    startProcess(options){
        const script = resolve(__dirname,options.path),
              child = cp.fork(script,[]);
        let invoked = false;
        child.on('message',(data)=>{
            options.message(data);
        })      
        child.on('exit',(code)=>{
            if(invoked){
                return
            }
            invoked = true;
            console.log(code);
        })
        child.on('error',(code)=>{
            if(invoked){
                return
            }
            invoked = true;
            console.log(code);
        })
    },
    qiniuUpload(options){
        const mac = new Qiniu.auth.digest.Mac(options.ak,options.sk),
              conf = new Qiniu.conf.Config(),
              client = new Qiniu.rs.BucketManager(mac,conf),
              key = nanoId()+options.ext;
        return new Promise((resolve,reject)=>{
            client.fetch(options.url,options.bucket,key,(error,ret,info)=>{
                if(error){
                    reject(error);
                }else{
                    if(info.statusCode === 200){
                        resolve({key})
                    }else{
                        reject(info)
                    }
                }
            })
        })      
    }
}      

  

4.控制器中

const {startProcess,qiniuUpload} = require('../lib/util'),
      config = require('../config/config')

class Crawler{  
    crawlSliderData(){
        startProcess({
            path:'../crawler/slider',
            async message(data){
                data.map(async item=>{
                    if(item.imgUrl && item.img_key){
                        const qiniu = config.qiniu;
                        try {
                            const imgData = await qiniuUpload({
                                url:item.imgUrl,
                                bucket:qiniu.bucket.tximg.bucket_name,
                                ak:qiniu.keys.ak,
                                sk:qiniu.keys.sk,
                                ext:'.jpg'
                            })
                            if(imgData.key){
                                item.imgKey = imgData.key
                            }
                        } catch (error) {
                            console.log(error)
                        }
                    }
                    console.log(data)
                })
            },
            async exit(data){
                console.log(data);
            },
            async error(data){
                console.log(data);
            }
        })
    }

}
module.exports=new Crawler();

5.slider.js

const dataItem = {
                cid: $elLink.attr('data-id'),
                href: $elLink.prop('href'),
                imgUrl: $elLink.find('img').prop('src'),
                title: $elLink.prop('title'),
                imgKey:''
            }

刷新上传

五.MySQL与Sequelize连接创建

1.安装sequelize与mysql2依赖

npm i mysql2 sequelize -S

2.在config文件夹中建立db_config.js

module.exports ={
    mysql:{
        base:{
            host:'localhost',
            dialect:'mysql',
            pool:{
                max:5,
                min:0,
                idle:10000
            }
        },
        conf:['txclass','root','123456']
    }
}

3.在crawler文件夹中再建立connection文件夹再建立mysql_connect.js文件

const Sequelize = require('sequelize'),
      {mysql} = require('../../config/db_config');
const seq = new Sequelize(...mysql.conf,mysql.base);
seq.authenticate().then(()=>{
    console.log('MySql server is connected completely');
}).catch((error)=>{
    console.log('MySql server is failed to be connected Error:'+error);
})
module.exports = seq;

4.node crawler/connection/mysql_connect.js调用

六.创建表模型、同步模型、数据入表

1.config下建立db_type_config.js

const Sequelize = require('sequelize');
module.exports={
    STRING: Sequelize.STRING,
    INT: Sequelize.INTEGER
}

2.db文件夹下connection文件夹下再建立models文件夹，再建立slider.js文件

const Sequelize = require('sequelize'),
      seq = require('../connection/mysql_connect'),
      {STRING,INT} = require('../../config/db_type_config');
const Slider = seq.define('slider',{
    cid:{
        comment:'course id',
        type: INT(25),
        allowNull:false,
        unique:true
    },
    href:{
        comment:'course detail page link',
        type: STRING,
        allowNull:false,
    },
    imgUrl:{
        comment:'course image url',
        type: STRING,
        allowNull:false,
    },
    title:{
        comment:'course name',
        type: STRING,
        allowNull:false,
    },
    imgKey:{
        comment:'course image name',
        type: STRING,
        allowNull:false,
    },
    status:{
        comment:'course status',
        type: INT,
        defaultValue: 1,
        allowNull:false
    }
    
})      
module.exports = Slider;

3.db的modules文件夹下建立index.js

const Slider = require('./slider');
module.exports={
    Slider
}

4.db目录下写一个sync.js

const seq = require('./connection/mysql_connect');
require ('./models');
seq.authenticate().then(()=>{
    console.log('MySql server is connected completely');
}).catch((error)=>{
    console.log('MySql server is failed to be connected Error:'+error);
})
seq.sync({
    force:true
}).then(()=>{
    console.log('The table has been synchronised into database successful')
    process.exit();
})

执行  node do/sync.js 建立表

5.建立service文件夹创建slider.js

const SliderModel = require('../do/models/slider');
const Slider = require('../do/models/slider');
class SliderService{
    async addSliderData(data){
        return await Slider.create(data)
    }
}
module.exports = new SliderService();

6.controller文件夹crawler.js中

const {startProcess,qiniuUpload} = require('../lib/util'),
      config = require('../config/config'),
      {addSliderData} = require('../service/slider')

class Crawler{  
    crawlSliderData(){
        startProcess({
            path:'../crawler/slider',
            async message(data){
                data.map(async item=>{
                    if(item.imgUrl && !item.img_key){
                        const qiniu = config.qiniu;
                        try {
                            const imgData = await qiniuUpload({
                                url:item.imgUrl,
                                bucket:qiniu.bucket.tximg.bucket_name,
                                ak:qiniu.keys.ak,
                                sk:qiniu.keys.sk,
                                ext:'.jpg'
                            })
                            if(imgData.key){
                                item.imgKey = imgData.key
                            }
                            const result = await addSliderData(item);
                            if(result){
                                console.log('Data create Ok')
                            }else{
                                console.log('Data create failed')
                            }
                        } catch (error) {
                            console.log(error)
                        }
                    }
                    console.log(data)
                })
            },
            async exit(data){
                console.log(data);
            },
            async error(data){
                console.log(data);
            }
        })
    }

}
module.exports=new Crawler();

网址访问执行