curl 抓取实例

 

function get_curl_info($url) {
/*$proxy = array(
"114.127.246.36:80",
"199.197.2.183:80",
"75.126.176.161:80",
"218.75.100.114:8080",
"60.12.190.58:8080",
"218.14.227.197:3128",
"cdn-kmr-c2100-01.tm.net.my:3128",
"119.70.40.101:8080",
"67.69.254.243:80",
"89.207.246.129:3128",
"190.144.127.202:3128"
);
for ($i=0;$i<strlen($proxy);$i++){*/
  $curl = curl_init();
  $header[0] = "Accept: text/xml,application/xml,application/xhtml+xml,";
  $header[0] .= "text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
  $header[] = "Cache-Control: max-age=0";
  $header[] = "Connection: keep-alive";
  $header[] = "Keep-Alive: 300";
  $header[] = "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7";
  $header[] = "Accept-Language: en-us,en;q=0.5";
  $header[] = "Pragma: ";
  //curl_setopt($curl, CURLOPT_PROXY, $proxy[$i]);
  curl_setopt($curl, CURLOPT_URL, $url);
  curl_setopt($curl, CURLOPT_USERAGENT, 'Googlebot/2.1 (+http://www.google.com/bot.html)');
  curl_setopt($curl, CURLOPT_HTTPHEADER, $header);
  curl_setopt($curl, CURLOPT_REFERER, 'http://www.google.com');
  curl_setopt($curl, CURLOPT_ENCODING, 'gzip,deflate');
  curl_setopt($curl, CURLOPT_AUTOREFERER, true);
  curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
  curl_setopt($curl, CURLOPT_TIMEOUT, 10);
  $html = curl_exec($curl);
  curl_close($curl);
/*  if($html!=false){
  break;
 }
}*/
  return $html;
}