爬虫自动化爬取单机游戏(nodejs)

C_and_H

已于 2023-08-09 10:52:42 修改

阅读量874

点赞数 1

文章标签：爬虫游戏 node.js

于 2023-08-04 15:16:53 首次发布

本文链接：https://blog.csdn.net/C_and_H/article/details/132104131

版权

闲话少说上代码!

这次我们以4399游戏为爬取对象冰娃与火娃6_冰娃与火娃6html5游戏在线玩_4399h5游戏-4399在线玩

我们将这些代码放入我们所需要爬取的页面中,由于一些游戏页面具有反爬虫技术

我们从构造器下手,从而避开无限调用debugger;

var temp=Function.prototype.constructor;

Function.prototype.constructor=function(x){
    if(x !='debugger'){
        return temp(x);
    }
    else{
        return function(){};
    }
}

其次我们需要分析这个游戏是在iframe还是在top上，显而易见游戏是在iframe上，因此我们需要把top切换成flash22

接下来需要启动一下本地服务器，让他们的服务器给我们本地服务器上传递数据

第一个文件主要用于接收我们需要爬取的服务器发来的信息，第二个文件主要用户读取里面的信息，并且生成对应的文件，当然我们需要配置跨域，这里我们采用cors。通过body-parser我们可以接收到服务器传来的引用数据类型数据，我们通过第一个代码可以得到host，然后再返回出去;

const express = require('express')
const app = express();
const bodyParser = require('body-parser');
const cors = require('cors')
const { start } = require('./appPlus')
const router = express.Router();


app.use(cors());
app.use(bodyParser.json({ extended: true }));

router.post('/sprider', sprider);

function sprider(req, res) {
    var { url, rootDir, publicPath, host } = req.body;
    if (publicPath =="") {
        let {public, HOST, links } = processingLinks(url);
        publicLength = public.split('/').length - 1;
        let reuslt = start(links, publicLength, rootDir);
        if (reuslt) {
            console.log("文件写入成功");
            res.end(JSON.stringify({ publicPath:public, host:HOST }));
        }
    }else{
        publicLength = publicPath.split('/').length - 1;
        let reuslt = start(url, publicLength, rootDir);
        if (reuslt) {
            console.log("文件写入成功");
            res.end(JSON.stringify({ publicPath, host }));
        }
    }

}

// Processing of URL links
function processingLinks(url) {
    let host = findHost(url);
    url = filterLinks(url, host);
    let publicPath = findPublicPath(url)
    return { public:publicPath, HOST:host, links:url };
}

// find host
function findHost(url) {
    let result = [];
    let times = [];
    for (let i = 0; i < url.length; i++) {
        let path = url[i].split("//")[1].split("/")[0];
        let index = result.indexOf(path);
        if (index >= 0) times[index]++;
        else {
            result.push(path);
            times.push(1);
        }
    }
    var max = Math.max(...times);
    let lastIndex = times.indexOf(max);
    return result[lastIndex];
}

// filter/fill links
function filterLinks(url, host) {
    let links = [];
    for (let i = 0; i < url.length; i++) {
        if (url[i].includes(host)) {
            if (url[i].endsWith('/')) url[i] += "index.html";
            links.push(url[i]);
        }
    }
    return links;
}

// find publicPath
function findPublicPath(url) {
    let [a, ...b] = url;
    let result = '';
    for (let i in a) {
        let flag = b.every(item => item[i] === a[i]);
        if (flag) result += a[i];
        else break;
    }
    return result;
}

app.use(router);

app.listen(3007, () => {
    console.log('http://127.0.0.1:3007')
})

const axios = require('axios');
const path = require('path')
const fs = require('fs')
const mkdirs = require(path.join(__dirname, './utils/mkdirs'))

const requests = axios.create({
    timeout: 60 * 1000,
    maxBodyLength: 100000000,
    Headers: {
        Referer: "http://www.1000mines.com/"
    }
});

// 请求资源的URL
const start = (url, length, rootDir) => {
    for (let i = 0; i < url.length; i++) {
        downloadFile(url[i], length, rootDir)
    }
    return true;
}

const downloadFile = async (url, length, rootDir) => {
    try {
        let response = await requests.get(url, { responseType: 'stream' })
        const urlArr = url.split('?')[0].split('/')
        let filePath = rootDir + urlArr.slice(length).join('/');
        filePath = filePath.replace(/\\/g, '/')
        console.log('filePath', filePath)
        mkdirs(filePath.slice(0, filePath.lastIndexOf('/')))
        response.data.pipe(fs.createWriteStream(path.join(__dirname, filePath)));
    } catch (e) {
        console.error("报错啦",e.config.url);
    }
}

exports.start = start;

最关键的地方到了，我们需要把下面的代码复制到控制台上，当然我们需要通过node去启动我们的本地服务器，例如你的文件名叫 serve.js 我们就node ./serve.js 就可以启动服务器了；这里我们定义根路径rootDir为test/；我们通过setInterval函数去监视随着游戏进度而进一步加载的网络请求，从而保证游戏的完整性;

var url = [];
let host,publicPath="";
var length=0;

(function () {
    if (!window.performance && !window.performance.getEntries) return false;
    length=window.performance.getEntries().length;
    window.performance.getEntries().forEach(async (item) => {
        if (item.name.includes(".") && !item.name.includes("127.0.0.1")) {
            if (item.responseStatus == undefined || item.responseStatus == 200) {
                if (!item.name.includes('google') && !item.name.includes("Google")) url.push(item.name);
            }
        }
    });
})();


function sendMessage(obj,links='http://127.0.0.1:3007/sprider'){
    if (obj.url.length > 0) {
        fetch(links, {
            method: "POST",
            headers: {
                "Content-Type": "application/json",
            },
            body: JSON.stringify(obj)
        }).then(res=>res.json()).then(res=>{
            host=res.host;
            publicPath=res.publicPath;
            console.log(host,publicPath);
        })
    }
}

sendMessage({url, rootDir:"test/",publicPath,host});


setInterval(()=>{
    var newLength=window.performance.getEntries().filter(v=>v.name.includes('127.0.0.1')).length;
    if(newLength>length){
        var arr=window.performance.getEntries().slice(length);
        arr= arr.filter(v=>v.name.includes(host));
        sendMessage({url, rootDir:"test/",publicPath,host})
        length=newLength;
    }
},2000)

由于4399需要配置对应的请求头，我们并没有去特意配置导致我们请求回来的index.html会出现错误,因为我们可以直接去resource(源代码)中复制index.html文件,若无法显示，刷新后即可显现

最后通过vscode live-server 插件

我们就可以获取到这个游戏

这只是一个小小的demo而已。最后附上我们的package.json文件和mkdirs文件.

{
  "name": "nodejs",
  "version": "1.0.0",
  "description": "",
  "main": "app.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "axios": "^1.4.0",
    "body-parser": "^1.20.2",
    "cors": "^2.8.5",
    "express": "^4.18.2",
    "uuid": "^9.0.0"
  }
}

const fs = require('fs')
const path = require('path')

const mkdirs = (dirname) => {
	if (fs.existsSync(dirname)) {
		return true;
	} else {
		if (mkdirs(path.dirname(dirname))) {
			fs.mkdirSync(dirname);
			return true;
		}
	}
}

// 递归创建目录
module.exports = mkdirs