php常用的几种采集方法

最新推荐文章于 2024-07-30 21:02:40 发布

魅力丛林

最新推荐文章于 2024-07-30 21:02:40 发布

阅读量564

点赞数

文章标签： php 开发语言服务器

本文链接：https://blog.csdn.net/MeiLiCongLin/article/details/125653905

版权

function curl_get($url) {

$ch = curl_init();

curl_setopt($ch, CURLOPT_URL, $url);

curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);

curl_setopt($ch, CURLOPT_HEADER, 1);

$result = curl_exec($ch);

$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);

if ($code != '404' && $result) {

return $result;

}

curl_close($ch);

}

//获取页面url链接

function get_page_urls($spider_page_result, $base_url) {

$get_url_result = preg_match_all("/\'\"\ ]*).*?>/", $spider_page_result, $out);

if ($get_url_result) {

return $out[1];

} else {

return;

}

}

/**模拟百度蜘蛛采集**/
function _GetContent( $url ){
        $ch = curl_init();
        $ip = '220.181.108.91';  // 百度蜘蛛
        $timeout = 15;
        curl_setopt($ch,CURLOPT_URL,$url);
        curl_setopt($ch,CURLOPT_TIMEOUT,0);
        //伪造百度蜘蛛IP
        curl_setopt($ch,CURLOPT_HTTPHEADER,array('X-FORWARDED-FOR:'.$ip.'','CLIENT-IP:'.$ip.''));
        //伪造百度蜘蛛头部
        curl_setopt($ch,CURLOPT_USERAGENT,"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)");
        curl_setopt($ch,CURLOPT_RETURNTRANSFER,1);
        curl_setopt($ch,CURLOPT_HEADER,0);
        curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,$timeout);
        $content = curl_exec($ch);
        if($content === false)
        {//输出错误信息
            $no = curl_errno($ch);
            switch(trim($no))
            {
                case 28 : $error = '访问目标地址超时'; break;
                default : $error = curl_error($ch); break;
            }
            echo $error;
        }
        else
        {
            $succ = true;
            return $content;
        }
    }

function post($url, $data = array())
{
	global $nochange;
	$o = "";
	foreach ($data as $k => $v) {
		$o .= "$k=" . $v . "&";
	}
	$data = substr($o, 0, -1);
	$ch = curl_init();
	curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
	curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
	curl_setopt($ch, CURLOPT_URL, $url);
	curl_setopt($ch, CURLOPT_ENCODING, 'gzip,deflate');
	curl_setopt($ch, CURLOPT_REFERER, $url);
	curl_setopt($ch, CURLOPT_USERAGENT, $url);
	curl_setopt($ch, CURLOPT_HEADER, false);
	curl_setopt($ch, CURLOPT_HTTPHEADER, array('CLIENT-IP:' . get_rand_ip(), 'X-FORWARDED-FOR:' . get_rand_ip()));
	curl_setopt($ch, CURLOPT_TIMEOUT, 30);
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
	curl_setopt($ch, CURLOPT_POST, 1);
	curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
	$rs = curl_exec($ch);
	$info = curl_getinfo($ch);
	curl_close($ch);
	$content_type = $info['content_type'];
	header('content-type:' . $content_type);
	if (nochange_url($content_type, $nochange_url) < 1) {
		$rs = change_link($rs);
		$rs = regstr($rs);
	}
	return $rs;
}

unction get($url)
{
	global $user_curl, $user_agent, $user_client, $nochange;
	if (!($cache = cache('r', $url))) {
		if (function_exists('curl_init') && $user_curl == "1") {
			$ch = curl_init();
			if ($user_agent == 'baidu') {
				if ($user_client == "mobile") {
					$user_agent = 'Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)';
				} elseif (is_mobile() && $user_client == "auto") {
					$user_agent = 'Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)';
				} elseif ($user_client == "pc" || !is_mobile()) {
					$user_agent = 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)';
				} else {
					$user_agent = 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)';
				}
			} elseif ($user_agent == 'google') {
				if ($user_client == "mobile") {
					$user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
				} elseif (is_mobile() && $user_client == "auto") {
					$user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
				} elseif ($user_client == "pc" || !is_mobile()) {
					$user_agent = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
				} else {
					$user_agent = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
				}
			} elseif ($user_agent == 'yahoo') {
				$user_agent = 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)';
			} elseif ($user_agent == 'bing') {
				if ($user_client == "mobile") {
					$user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)';
				} elseif (is_mobile() && $user_client == "auto") {
					$user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)';
				} elseif ($user_client == "pc" || !is_mobile()) {
					$user_agent = 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)';
				} else {
					$user_agent = 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)';
				}
			} else {
				$user_agent = 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)';
			}
			curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
			curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
			curl_setopt($ch, CURLOPT_URL, $url);
			curl_setopt($ch, CURLOPT_ENCODING, 'gzip,deflate');
			curl_setopt($ch, CURLOPT_REFERER, $url);
			curl_setopt($ch, CURLOPT_USERAGENT, $user_agent);
			curl_setopt($ch, CURLOPT_HEADER, false);
			curl_setopt($ch, CURLOPT_HTTPHEADER, array('CLIENT-IP:' . get_rand_ip(), 'X-FORWARDED-FOR:' . get_rand_ip()));
			curl_setopt($ch, CURLOPT_TIMEOUT, 30);
			curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
			curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
			$rs = curl_exec($ch);
			$info = curl_getinfo($ch);
			curl_close($ch);
			$content_type = $info['content_type'];
		} else {
			$rs = file_get_contents($url);
		}
		if (nochange_url($content_type, $nochange) < 1 && nochange_url($content_type, "jpg|jpeg|gif|png|bmp") < 1) {
			$rs = change_link($rs);
			$rs = regstr($rs);
		}
		if (nochange_url($content_type, "jpg|jpeg|gif|png|bmp") < 1) {
			$cache = array('content_type' => $content_type, 'rs' => $rs);
			cache('w', $url, $cache);
		}
		header('content-type:' . $content_type);
		return $rs;
	} else {
		extract($cache);
		header('content-type:' . $content_type);
		return $rs;
	}
}