Koa2 + Puppeteer打造『爬虫系统』4

十.创建推荐课程表模型以及数据入表操作

1.db_type_config中加入小数类型

const Sequelize = require('sequelize');
module.exports={
    STRING: Sequelize.STRING,
    INT: Sequelize.INTEGER,
    DECIMAL:Sequelize.DECIMAL
}

2.在modles中创建recomCourse.js

const seq = require('../connection/mysql_connect'),
      {STRING,INT,DECIMAL} = require('../../config/db_type_config');
const RecomCourse = seq.define('recom_course',{
    cid:{
        comment:'course ID',
        type:INT,
        allowNull:false,
        unique:true
    },
    href:{
        comment:'course detail page link',
        type:STRING,
        allowNull:false,
    },
    mainTitle:{
        comment:'Page category title',
        type:STRING,
        allowNull:false
    },
    title:{
        comment:'course name',
        type:STRING,
        allowNull:false
    },
    posterUrl:{
        comment:'course poster image',
        type:STRING,
        allowNull:false
    },
    description:{
        comment:'course description',
        type:STRING,
        allowNull:false
    },
    teacherImg:{
        comment:'teacher image',
        type:STRING,
        allowNull:false
    },
    teacherName:{
        comment:'teacher name',
        type:STRING,
        allowNull:false
    },
    studentCount:{
        comment:'count of students ',
        type:INT,
        allowNull:false
    },
    price:{
        comment:'course price',
        type:DECIMAL,
        allowNull:false
    },
    posterKey:{
        comment:'qiniu course image name',
        type:STRING,
        allowNull:false
    },
    teacherImgKey:{
        comment:'qiniu teacher image name',
        type:STRING,
        allowNull:false
    }

})
module.exports = RecomCourse

3.入口导入

const Slider = require('./slider'),
      RecomCourse = require('./recomCourse'),
      AgencyInfo = require('./agencyinfo')

module.exports={
    Slider,AgencyInfo,RecomCourse
}

4.service中新建RecomCourse.js

const RecomCourseModel = require('../do/models/recomCourse');
class RecomCourseService{
    async addRecomCourse(data){
        const cid = data.cid;
        const result = await RecomCourseModel.findOne({
            where:{cid}
        })
        if(result){
            return await RecomCourseModel.update(data,{
                where:{
                    cid
                }
            })
        }else{
            return await RecomCourseModel.create(data);
        }
    }
}
module.exports = new RecomCourseService();

5.controler中增加

 {addRecomCourse} = require('../service/recomCourse'),
       const result = await addAgencyInfo(data);
                        if(result){
                            console.log('Data create Ok')
                        }else{
                            console.log('Data create failed')
                        }

6.node do/sync.js创建表，http://localhost:3000/crawler/crawl_recom_course写入

十一.爬取课程集合列表以及上传七牛图床

1.crawlers文件夹中新建collection.js

const Crawler = require('../lib/crawler'),
    { crawler } = require('../config/config');
Crawler({
    url: crawler.url.main,
    callback() {
        const $ = window.$,
            $item = $('.agency-recommend-course');
        let data = [];
        $item.each((index,item)=>{
            const $el = $(item);
            const dataItem = {
                cid: index +1,
                title: $el.find('.recommend-course-title span').eq(0).text().replace(/(\\n|\s+|更多)/g,''),
                info: $el.find('.rec-group-info').text(),
                qqQunLink:$el.find('.rec-group-join').prop('href'),
                posterUrl:$el.find('.rec-group-mask').css('background-image').match(/\"(.+?)\"/)[1],
                courseIdList:'',
                posterKey:''
            }
            let _idList = [];
            const $courseItem = $el.find('.course-card-item');
            $courseItem.each((index,item)=>{
                const $elem = $(item);
                _idList.push($elem.find('.item-img-link').attr('data-id'));
            })
            dataItem.courseIdList = _idList.toString();
            data.push(dataItem);
        })
        return data;
    }
})      

2.定义路由

const router = require('koa-router')(),
      crawlerController = require('../controller/crawler')
router.prefix('/crawler')
router.get('/crawl_slider_data',crawlerController.crawlSliderData )
router.get('/crawl_agencyr_info',crawlerController.crawlAgencyInfo )
router.get('/crawl_recom_course',crawlerController.crawlRecomCourse )
router.get('/crawl_collection',crawlerController.crawlCollection )

module.exports = router

3.controller定义函数

其中先将addCollection函数写上了

crawlCollection(){
        startProcess({
            path:'../crawler/collection',
            async message(data){
               
                data.map(async item=>{
                   try {
                       const qiniu = config.qiniu;
                       if(item.posterUrl && !item.posterKey){
                           const posterData = await qiniuUpload({
                               url:item.posterUrl,
                               bucket:qiniu.bucket.tximg.bucket_name,
                               ext:'.jpg'
                           })
                           if(posterData.key){
                               item.posterKey = posterData.key
                           }
                       }
                       const result = await addCollection(item);
                       if(result){
                           console.log('Data create Ok')
                       }else{
                           console.log('Data create failed')
                       }
                   } catch (error) {
                       console.log(error)
                   }
                    
                })
            },
            async exit(data){
                console.log(data);
            },
            async error(data){
                console.log(data);
            }
        })
    }

十二.创建课程集合表模型以及入表操作

1.db的modules中建立collection.js模型

const seq = require('../connection/mysql_connect'),
   { STRING,INT } = require('../../config/db_type_config');
const Collection = seq.define('collection',{
    cid:{
        comment: 'collection ID',
        type:INT,
        allowNull:false,
        unique:true
    },
    title:{
        comment: 'collection title',
        type:STRING,
        allowNull:false
    },
    info:{
        comment:'collection info',
        type:STRING,
        allowNull:false
    },
    qqQunLink:{
        comment:'collection qqlink',
        type:STRING,
        allowNull:false
    },
    posterUrl:{
        comment:'poster image link',
        type:STRING,
        allowNull:false
    },
    courseIdList:{
        comment:'the collection for course ID',
        type:STRING,
        allowNull:false
    },
    posterKey:{
        comment:'qiniu poster image name',
        type:STRING,
        allowNull:false
    }
    
})   
module.exports = Collection;

2.出口文件导入

const Slider = require('./slider'),
      RecomCourse = require('./recomCourse'),
      AgencyInfo = require('./agencyinfo'),
      Collection = require('./collection')

module.exports={
    Slider,AgencyInfo,RecomCourse,Collection
}

3.services文件夹中建立collection.js

const CollectionModel = require('../do/models/collection');
class CollectionService{
    async addCollection(data){
        const cid = data.cid;
        const result = await CollectionModel.findOne({
            where:{cid}
        })
        if(result){
            return await CollectionModel.update(data,{
                where:{cid}
            })
        }else{
            return await CollectionModel.create(data)
        }
    }
}
module.exports = new CollectionService();

4.控制器中调用,并将数据传入方法址姓

      {addCollection} = require('../service/collection')

5.终端调用 node db/sync.js创建表

6.访问写入数据

http://localhost:3000/crawler/crawl_collection