php curl 需要登录爬取页面

为什么80%的码农都做不了架构师？>>>   

$data='username=demo_peter@126.com&password=123qwe&remember=1';
$curlobj = curl_init();          // 初始化
curl_setopt($curlobj, CURLOPT_URL, "http://www.imooc.com/user/login");    // 设置访问网页的URL
curl_setopt($curlobj, CURLOPT_RETURNTRANSFER, true);         // 执行之后不直接打印出来

// Cookie相关设置，这部分设置需要在所有会话开始之前设置
date_default_timezone_set('PRC'); // 使用Cookie时，必须先设置时区
curl_setopt($curlobj, CURLOPT_COOKIESESSION, TRUE);

curl_setopt($curlobj,CURLOPT_COOKIEFILE,"cookiefile");
curl_setopt($curlobj,CURLOPT_COOKIEJAR,"cookiefile");
curl_setopt($curlobj,CURLOPT_COOKIE,session_name().'='.session_id());

curl_setopt($curlobj, CURLOPT_HEADER, 0); 
// 注释掉这行，因为这个设置必须关闭安全模式 以及关闭open_basedir，对服务器安全不利
//curl_setopt($curlobj, CURLOPT_FOLLOWLOCATION, 1);  

curl_setopt($curlobj, CURLOPT_POST, 1);  
curl_setopt($curlobj, CURLOPT_POSTFIELDS, $data);  
curl_setopt($curlobj, CURLOPT_HTTPHEADER, array("application/x-www-form-urlencoded; charset=utf-8", 
   "Content-length: ".strlen($data)
   )); 
curl_exec($curlobj);   // 执行
curl_setopt($curlobj, CURLOPT_URL, "http://www.imooc.com/space/index");
curl_setopt($curlobj, CURLOPT_POST, 0);  
curl_setopt($curlobj, CURLOPT_HTTPHEADER, array("Content-type: text/xml"
   )); 
$output=curl_redir_exec($curlobj); // 执行
curl_close($curlobj);        // 关闭cURL
echo $output;

/**
 * 自定义实现页面链接跳转抓取
 */
function curl_redir_exec($ch,$debug="") 
{ 
    static $curl_loops = 0; 
    static $curl_max_loops = 20; 

    if ($curl_loops++ >= $curl_max_loops) 
    { 
        $curl_loops = 0; 
        return FALSE; 
    } 
    curl_setopt($ch, CURLOPT_HEADER, true); // 开启header才能够抓取到重定向到的新URL
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); 
    $data = curl_exec($ch); 
    // 分割返回的内容
    $h_len = curl_getinfo($ch, CURLINFO_HEADER_SIZE); 
    $header = substr($data,0,$h_len);
    $data = substr($data,$h_len - 1);

    $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); 
    if ($http_code == 301 || $http_code == 302) { 
        $matches = array(); 
        preg_match('/Location:(.*?)\n/', $header, $matches); 
        $url = @parse_url(trim(array_pop($matches))); 
        // print_r($url); 
        if (!$url) 
        { 
            //couldn't process the url to redirect to 
            $curl_loops = 0; 
            return $data; 
        } 
        $last_url = parse_url(curl_getinfo($ch, CURLINFO_EFFECTIVE_URL)); 
        if (!isset($url['scheme'])) 
            $url['scheme'] = $last_url['scheme']; 
        if (!isset($url['host'])) 
            $url['host'] = $last_url['host']; 
        if (!isset($url['path'])) 
            $url['path'] = $last_url['path'];

        $new_url = $url['scheme'] . '://' . $url['host'] . $url['path'] . (isset($url['query'])?'?'.$url['query']:''); 
        curl_setopt($ch, CURLOPT_URL, $new_url); 

        return curl_redir_exec($ch); 
    } else { 
        $curl_loops=0; 
        return $data; 
    } 
}

1. <?php  
2.  
3.     $url = 'http://zhidao.chanjet.com/restserver/zhidao';  
4.     $data = array(  
5.         'api_key'=>'3qQ2Edm62Vd4bAVCwNoxgn0l',  
6.         'method'=>'baidu.zhidao.getQuestionList',  
7.         'call_id'=>'1308713190',  
8.         'cid'=>59533,  
9.         'qstatus'=>1,  
10.         'format'=>'json',  
11.         'page_no'=>1,  
12.         'page_size'=>25,  
13.         'keywords'=>'财务',  
14.         'bd_sig'=>'2bad1c47bb75e0363a689f4b09743afb'  
15.     );  
16.   
17.     $json_data = postData($url, $data);  
18.     $array = json_decode($json_data,true);  
19.     echo '<pre>';print_r($array);  
20.       
21.     function postData($url, $data)  
22.     {  
23.         $ch = curl_init();  
24.         $timeout = 300;   
25.         curl_setopt($ch, CURLOPT_URL, $url);  
26.         curl_setopt($ch, CURLOPT_POST, true);  
27.         curl_setopt($ch, CURLOPT_POSTFIELDS, $data);  
28.         curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);  
29.         curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);  
30.         $handles = curl_exec($ch);  
31.         curl_close($ch);  
32.         return $handles;  
33.     }  
34.   
35. ?>  

 

 

$curlobj = curl_init();
curl_setopt($curlobj, CURLOPT_URL, "https://news.zhibo8.cc/nba/");
curl_setopt($curlobj, CURLOPT_POST, 0);
curl_setopt($curlobj, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($curlobj, CURLOPT_HTTPHEADER, array("Content-type: text/xml"
));
curl_setopt($curlobj, CURLOPT_SSL_VERIFYPEER, false);
$output=curl_exec($curlobj); // 执行
       // 关闭cURL
if ($output === FALSE) {
    echo "cURL Error: " . curl_error($curlobj);
}
curl_close($curlobj);
//var_dump($output);

preg_match('/<table[^>]*>(.*)<\/table>/is', $output, $arMatch);
var_export($arMatch);

转载于:https://my.oschina.net/SimTao/blog/1815067