基于node.js实现的爬虫

1.连接数据库

// 连接数据库

const { Sequelize } = require('sequelize');

const sequelize = new Sequelize('myschool', 'root', 'root', {
  host: 'localhost',
  dialect: 'mysql'
});


// 创建一个数据库连接的实力依赖的是连接池，所以不需要我们手动关闭数据库连接
// 有场景需要可以使用sequelize.close()进行关闭


module.exports = sequelize;

2.创建数据库相关表

const sequelize = require("./db.js")
const { DataTypes } = require('sequelize');
// id，名称，图片，出版时间，作者
const Book = sequelize.define('Book', {
    name : {
        type : DataTypes.STRING,
        allowNull : false
    },
    imgUrl : {
        type : DataTypes.STRING
    },
    publishDate : {
        type : DataTypes.STRING,
        allowNull : false
    },
    author : {
        type : DataTypes.STRING,
        allowNull : false
    }
}, {
  paranoid : true,
});


module.exports = Book;

3,。同步用户模型表

// 用于同步所有模型
require("./Book")
const sequelize = require("./db");


(async function(){
    await sequelize.sync({ alter: true });
    console.log("用户模型表刚刚（重新）创建");
})()

4.对猫眼首页数据进行爬取

const axios = require("axios").default;
const cheerio = require("cheerio")
const Movie = require("../models/Movie")



async function getMoviesHTML(){
    const resp = await axios.get("https://maoyan.com/")
    // console.log(resp.data);
    return resp.data

}

async function getMoviesPingfen(){
    const resp = await axios.get("https://maoyan.com/")
    const $ = cheerio.load(resp.data)
    const pingfen1 =$("#app.container .content .main .movie-grid .panel .panel-content .movie-list .movie-item .movie-poster .movie-overlay .movie-info .movie-score i.integer")
    const pingfen2 = $("#app.container .content .main .movie-grid .panel .panel-content .movie-list .movie-item .movie-poster .movie-overlay .movie-info .movie-score i.fraction")
    const pingfens = []
    pingfen1.each((index,item) => {
        pingfens.push($(item).text()+pingfen2.eq(index).text())
    })
    // console.log(arr)
    return pingfens
    
}
// getMoviesPingfen()


async function getMoviesLinks(){
    const html = await getMoviesHTML()
    const $ = cheerio.load(html)//使用cheerio库
    // console.log($)
    const achorElements=$("#app.container .content .main .movie-grid .panel .panel-content .movie-list .movie-item>a")
    // console.log(achorElements[0])
    const links = achorElements.map((i,ele)=>{
        const href = ele.attribs["href"];
        return href;
    }).get()
    // console.log(links)
    return links
}
// getMoviesLinks()



// 3.根据数据的详情页地址进入详情页，得到电影的详细信息
async function getMovieDetail(detailUrl,pingfen){
    console.log(pingfen);
    const resp = await axios.get(`https://www.maoyan.com${detailUrl}`)
    const $ = cheerio.load(resp.data)
    const name = $(".banner .wrapper .celeInfo-right .movie-brief-container h1").text().trim()
    // console.log('111');
    const time1 = $(".banner .wrapper .celeInfo-right .movie-brief-container ul .ellipsis").eq(1).text().trim();
    const time2 =  time1.split("/")
    const time = time2[1].trim()
    const imgUrl = $(".banner .wrapper .celeInfo-left .avatar-shadow  img.avatar").attr('src');
    const buildDate = $(".banner .wrapper .celeInfo-right .movie-brief-container ul .ellipsis").eq(2).text().substring(0,10).trim();
    return {
        name,
        imgUrl,
        buildDate,
        time,
        pingfen
    }
}
getMovieDetail("/films/70798")


// 4获取所有电影的信息
async function fetchAll(){
    const links = await getMoviesLinks();
    const fen=await getMoviesPingfen();
    // console.log(links);
    const proms = links.map((link,i)=>{
        // console.log(i);

        return getMovieDetail(link,fen[i])
    })
    // console.log(proms);
    return Promise.all(proms)
}


async function savaToDB(){
    const movies = await fetchAll()
    // console.log(books)
    await Movie.bulkCreate(movies)
    console.log("数据保存至数据库")
}
savaToDB()

5.数据库显示