PHP 使用simple_html_dom 类库爬取古诗文网的古诗数据

gezipml

于 2023-01-07 15:24:31 发布

阅读量214

点赞数

分类专栏： yii2 PHP 文章标签： php 开发语言

本文链接：https://blog.csdn.net/gezipml/article/details/128591379

版权

PHP 同时被 2 个专栏收录

11 篇文章 0 订阅

订阅专栏

yii2

2 篇文章 0 订阅

订阅专栏

使用yii2框架，执行命令行：php yii hello/html
开箱即食

public function actionHtml()
    {

        require_once __DIR__ . DIRECTORY_SEPARATOR .'../common/simple_html_dom.php';
        $page = 1;
        $url = 'https://so.gushiwen.cn/shiwens/default.aspx?page='.$page.'&tstr=&astr=%e6%9d%9c%e7%94%ab&cstr=&xstr=';
        $htmlObj = new \simple_html_dom();
        $htmlStr = file_get_contents($url);
        $cotObj = $htmlObj->load($htmlStr);
        $zzobj = $cotObj->find('#type2 .sright',0);
        $insertData = [];
        $allAuthor = FaPoetry::find()->select('author')->groupBy('author')->column();
        foreach ($zzobj->find('a') as $value){
            //作者
            $author = trim($value->plaintext);
            if (in_array($author,$allAuthor)) {
                continue;
            }
            $page = 1;
            $encodeAuthor = urlencode($author);
            while ($page <= 10){//网站只允许访问到第10页
                $url = 'https://so.gushiwen.cn/shiwens/default.aspx?page='.$page.'&tstr=&astr='.$encodeAuthor.'&cstr=&xstr=';
                $htmlObj = new \simple_html_dom();
                $htmlStr = file_get_contents($url);
                $cotObj = $htmlObj->load($htmlStr);
                foreach ($cotObj->find('#leftZhankai .sons .cont')  as $cont){
                    $insertItem = [];
                    $insertItem['title'] = $cont->find('p a b',0)->plaintext;

                    foreach ($cont->find('.source a') as $k=>$item){
                        if ($k == 0) {
                            $insertItem['author'] = $item->plaintext;
                            $isInsert = FaPoetry::findOne(['title'=>$insertItem['author']]);
                            if (!empty($isInsert)) {
                                break 3;
                            }

                        }else{
                            $insertItem['dynasty'] = $item->plaintext;
                        }
                    }
                    $content = $cont->find('.contson',0)->plaintext;
                    $insertItem['content'] = trim($content);
                    $insertItem['createtime'] = time();
                    $insertData[] = $insertItem;
                    if (count($insertData) == 100) {
                        \Yii::$app->db->createCommand()->batchInsert(FaPoetry::tableName(),['title','author','dynasty','content','createtime'],$insertData)->execute();
                        $insertData = [];
                    }
                }
                $page++;
            }
        }


        if (!empty($insertData)) {
            \Yii::$app->db->createCommand()->batchInsert(FaPoetry::tableName(),['title','author','dynasty','content','createtime'],$insertData);
        }
    }