curl采集

 

// $url = 'www.baidu.com';
$url = 'https://h5.eqxiu.com/s/CzvPVCNt';

function http_curl($url, $post = '', $cookie, $cookiejar, $referer = '')
{

    //使用透明代理时伪造ip
    $headers = forgeIp();
    //POST提交大数据（超过1024字节）异常解决方法
    $headers['Expect:'] = '';
    $options = [

        CURLOPT_URL => $url,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_HEADER => true,//返回内容包含响应头
        CURLOPT_TIMEOUT => 30,//执行时间
        //设置 HTTP 头字段的数组
        CURLOPT_HTTPHEADER => $headers,
        //模拟浏览器行为，在HTTP请求中包含一个"User-Agent: "头的字符串。
        CURLOPT_USERAGENT => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
        //返回 HTTP 头中的 "Location: " 重定向页面
        CURLOPT_FOLLOWLOCATION => true,
        //取消https请求的证书认证
        CURLOPT_SSL_VERIFYPEER => false,
        CURLOPT_SSL_VERIFYHOST => 0,
        //追踪请求头信息，用于后续curl_getinfo获取输出
        CURLINFO_HEADER_OUT => true

    ];

    $ch = curl_init();
    curl_setopt_array($ch, $options);

    //post请求
    if ($post) {
        curl_setopt($ch, CURLOPT_POST, true);
        curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($post));
    }

    //设定 HTTP 请求中Cookie
    if ($cookie) {
        curl_setopt($ch, CURLOPT_COOKIE, $cookie);
    }

    if ($cookiejar) {
        $cachePath = __DIR__ . '/cache/' . $cookiejar;
        //连接结束后，将cookie信息保存到文件中
        curl_setopt($ch, CURLOPT_COOKIEJAR, $cachePath);
        //包含 cookie 数据的文件名
        curl_setopt($ch, CURLOPT_COOKIEFILE, $cachePath);
    }

    if ($referer) {
        //在HTTP请求头中"Referer: "的内容。
        curl_setopt($ch, CURLOPT_REFERER, $referer);
    } else {
        // TRUE 时将根据 Location: 重定向时，自动设置 header 中的Referer:信息。
        curl_setopt($ch, CURLOPT_AUTOREFERER, true);
    }

    $data = curl_exec($ch);

    //执行之后获取请求头信息
    $request_headers = curl_getinfo($ch, CURLINFO_HEADER_OUT);

    if(curl_exec($ch) === false) {
        return $data;
    } else {
        echo 'CURL_ERRNO:' . curl_errno($ch);
        echo '<br>';
        echo 'CURL_ERROR:' . curl_error($ch);
    }

    curl_close($ch);

    return [
      'content' => $data,
      'request_headers' => $request_headers
    ];
}

//此函数提供了国内的IP地址
function forgeIp()
{
    $ip_long = array(
        array('607649792', '608174079'), //36.56.0.0-36.63.255.255
        array('1038614528', '1039007743'), //61.232.0.0-61.237.255.255
        array('1783627776', '1784676351'), //106.80.0.0-106.95.255.255
        array('2035023872', '2035154943'), //121.76.0.0-121.77.255.255
        array('2078801920', '2079064063'), //123.232.0.0-123.235.255.255
        array('-1950089216', '-1948778497'), //139.196.0.0-139.215.255.255
        array('-1425539072', '-1425014785'), //171.8.0.0-171.15.255.255
        array('-1236271104', '-1235419137'), //182.80.0.0-182.92.255.255
        array('-770113536', '-768606209'), //210.25.0.0-210.47.255.255
        array('-569376768', '-564133889'), //222.16.0.0-222.95.255.255
    );
    $rand_key = array_rand($ip_long);
    $ip = long2ip(mt_rand($ip_long[$rand_key][0], $ip_long[$rand_key][1]));
    $headers['CLIENT-IP'] = $ip;
    $headers['X-FORWARDED-FOR'] = $ip;

    $headerArr = array();
    foreach ($headers as $n => $v) {
        $headerArr[] = $n . ':' . $v;
    }
    return $headerArr;
}

$info = http_curl($url, '', 'cookie.txt', 'https://www.jstor.org/', '', true);
var_dump($info);