Node端爬取数据处理(http和https)

1. 安装 http 和 https

npm install http -S
npm install https -S

2. 爬取数据处理

const http = require('http');
const https = require('https');
	
const  getUrlData = async(url, type) => {
	let request = type === 'https' ? https : http;
		
	return new Promise((resolve, reject) => {
			request.get(url, (res) => {
				const { statusCode } = res;
				let error;
				if(statusCode !== 200){
					error = new Error(`请求失败. \n 状态码: ${statusCode}`);
				}
				if(error){
					console.log(error.message);
					//	消耗响应数据以释放内存
					res.resume();
					resolve();
				}
				
				res.setEncoding('utf8');
				let rawData = '';
				res.on('data', (chunk) => { rawData += chunk; });
				res.on('end', () => {
					resolve(rawData);
				})
			}).on("error", (e) => {
				console.error(`错误：${e.message}`);
				resolve();
			})
		})
	}

参考

相关api使用请参考 node 学习网站