百度搜索404页面

2019独角兽企业重金招聘Python工程师标准>>>

site:www.***.com 娱乐城

先用火车头采集404百度地址

再用php 取出真实地址

<?php
/** 百度404
error_reporting(0);
set_time_limit(0);
function getrealurl($url){
    $header = @get_headers($url,1);
    if (strpos($header[0],'301') || strpos($header[0],'302')) {
        if(is_array($header['Location'])) {
            $returnurl = $header['Location'][count($header['Location'])-1];
        }else{
            $returnurl = $header['Location'];
        }
        $header = @get_headers($returnurl,1);
        if(strpos($header[0],'404')){
            return $returnurl;
        }else{
            //echo 'No 404:'.$returnurl;
        }
    }else {
        //return $url;
    }
}
$handle = @fopen("404url.txt", "r");
if ($handle) {
    while (!feof($handle)) {
        $url = trim(fgets($handle));
        $realurl = false;
        while (!$realurl) {        
            $realurl = getrealurl($url);
            if($realurl){
                echo $realurl."<br />";
                 //sleep(1);
            }
        }           
    }
    fclose($handle);
}
exit();

转载于:https://my.oschina.net/u/937746/blog/285324