Node.js 网站内容抓取及Mysql存取Demo

var DBUtils = {
    //新增
    _insert: function (client, insertSQLString, value) {
        client.query(insertSQLString, value, function (error, results) {
            if (error) {
                console.log("ClientReady Error:" + error.message);
                client.end();
                return;
            } else {
                console.log("Inserted:" + results.affectedRows + " row.");
                console.log("Insert success...");

            }
        });
    },
    //查询
    _select: function (client, selectSQLString) {
        client.query(selectSQLString, function (error, results, fields) {
            if (error) {
                console.log("GetData Error:" + error.message);
                client.end();
                return;
            }

            if (results.length > 0) {
                var firstResult, resultSet = '';

                for (var i = 0; i < results.length; i++) {
                    firstResult = results[i];
                    resultSet += "ID:" + firstResult["ID"] + ' Name:' + firstResult["Name"] + " PWD:" + firstResult["PWD"] + "\n";
                }
                console.log(resultSet);
            }
        });
    },
    //更新
    _update: function (client, updateSQLString) {
        client.query(updateSQLString, function (error, results) {
            if (error) {
                console.log("ClientReady Error:" + error.message);
                client.end();
                return;
            }

            console.log("Update success...");
        });
    },
    //删除
    _delete: function (client, deleteSQLString) {
        client.query(deleteSQLString, function (error, results) {
            if (error) {
                console.log("ClientReady Error:" + error.message);
                client.end();
                return;
            }

            console.log("Delete success...");
        });
    }
};

exports._insert = DBUtils._insert;
exports._select = DBUtils._select;
exports._update = DBUtils._update;
exports._delete = DBUtils._delete;

 

///山西政府采购网公示招标信息爬取///
var request = require('request');
var cheerio = require('cheerio');
var db = require("./DBUtils.js")

var list_url = 'http://www.ccgp-shanxi.gov.cn/view.php?nav=100';
var view_url = 'http://www.ccgp-shanxi.gov.cn/view.php?nid='

var myDate = new Date();
//当前日期
var currDate = myDate.getFullYear()+"-"+myDate.getMonth()+"-"+myDate.getDate();
//来源
var from_ = "山西政府采购网";
//分类
var cate1 = "政采信息";

var mysql      = require('mysql');
var client = mysql.createConnection({
  host     : 'localhost',
  user     : 'app',
  password : 'app',
  database : 'test'
});

client.connect();

request(list_url, function (error, res, body) {
    if (!error && res.statusCode == 200) {
//        console.log("body:", body);
        var $ = cheerio.load(body);
        var trs = $('#node_list tbody tr');
        //循环tr
        var i = 0;
        trs.each(function(index,element){
            var startDate = $(this).children().children("#sdate"+i).val();
            var endDate = $(this).children().children("#fdate"+i).val();
            var title = $(this).children().children("a").attr("title");
            var viewUrl = $(this).children().children("a").attr("href");
            var orid = viewUrl.substr(viewUrl.indexOf("=")+1,viewUrl.length);
            viewUrl = view_url + orid;
            var area = $(this).children().next().text();
            
            db._insert(client, "insert into news set title=?,from_=?,detail=?,getDate=?,publicDate=?,remark=?,cate1=?", [title, from_, viewUrl,currDate, startDate,"结束于"+endDate, cate1]);  

            console.log("title==="+title);

            i++;
        });

        client.end();
    }
})

 

转载于:https://my.oschina.net/wowlinda/blog/550801