xx网站爬虫

利用simple_html_dom的采集数据实例，这是一个PHP的库，上手很容易。

下载地址：https://github.com/samacs/simple_html_dom

<?php
//软件分类
include_once 'simple_html_dom.php';
$conn=mysql_connect("127.0.0.1", "root", "root");
set_time_limit(0);
if (!$conn) {
  die('Could not connect: ' . mysql_error());
}
mysql_select_db("wifi_common" , $conn);
mysql_query("set names utf8");
$appHtml = file_get_html('http://www.wandoujia.com/apps');
$appData = $appHtml->find("#j-head-menu .app-popup li");
foreach($appData as $appKey => $eachAppData) {
    $allHref = $eachAppData->find("a", 0)->href;
    $html = file_get_html($allHref);
    $listData = $html->find('#j-tag-list li');
    foreach($listData as $key=>$eachRowData){    
        foreach ($eachRowData->find(".app-desc") as $key1=>$eachRowData1) {
            $appUrl=$eachRowData->find(".icon-wrap a",0)->href;
            $appUrlInfo = file_get_html($appUrl);
            $appUrlInfoDetail = $appUrlInfo->find(".detail-wrap");
            foreach ($appUrlInfoDetail as $info) {
               $subTitle = trim($info->find(".tagline", 0)->plaintext);//子标题
               $appName = trim($info->find(".title", 0)->plaintext);//appName
               $desc = trim($info->find(".desc-info .con", 0)->plaintext);
               $icon = trim($info->find(".app-icon img", 0)->src);
               $sizeM = trim($info->find(".infos .infos-list dd", 0)->plaintext);
               $size = $sizeM * 1000;
               $apkUrl = trim($info->find(".download-wp a", 0)->href);
               $explodeApk = explode("/", $apkUrl);
               $package = $explodeApk[4];
               $countSql = "select count(*) as count  from market where title='".$appName."'";
               $result =  mysql_fetch_array(mysql_query($countSql));
                if ($result['count'] > 0) {
                    continue;
                }
                $sql = "insert into market (`category`, `subCategory`, `title`, `subTitle`, `desc`, `icon`, `apk`, `size`, `package`) values 
(2, 0, '".$appName."', '".$subTitle."', '".$desc."', '".$icon."', '".$apkUrl."', '".$size."', '".$package."')";
                mysql_query($sql); <span style="font-family:Verdana, Geneva, Arial, Helvetica, sans-serif;">//把采集的数据添加到数据库
            }
        }
    }
}

在查看数据库,数据就添加上去了 豌豆荚数据比较多，所以采集起来有点慢