PHP爬虫之phpspider

1. composer require owner888/phpspider  下载包

2.自动化抓取

use phpspider\core\phpspider;
$configs = array( 'name' => '糗事百科', 'domains' => array( 'qiushibaike.com', 'www.qiushibaike.com' ), 'scan_urls' => array( 'http://www.qiushibaike.com/' ), 'content_url_regexes' => array( "http://www.qiushibaike.com/article/\d+" ), 'list_url_regexes' => array( "http://www.qiushibaike.com/8hr/page/\d+\?s=\d+" ), 'fields' => array( array( // 抽取内容页的文章内容 'name' => "article_content", 'selector' => "//*[@id='single-next-link']", 'required' => true ), array( // 抽取内容页的文章作者 'name' => "article_author", 'selector' => "//div[contains(@class,'author')]//h2", 'required' => true ), ), ); 

$spider = new phpspider($configs);//实例化

$spider->start(); ​

3.自定义抓取 + XPath选择

use phpspider\core\requests;   //phpspider下的requests
use phpspider\core\selector;        

 $url = 'https://xxx.com/r17678.html';

$html = requests::get($url);

$href = selector::select($html,'//div[@class="chapter"]/a/@href');