php 解析大json

最新推荐文章于 2024-06-27 17:56:13 发布

鲨鱼辣椒123456

最新推荐文章于 2024-06-27 17:56:13 发布

阅读量57

点赞数 2

文章标签： php json java

本文链接：https://blog.csdn.net/weixin_39560112/article/details/139863261

版权

<?php
/*
 * @Description: 
 * @Version: 2.0
 * @Autor: ABing
 * @Date: 2024-06-05 17:59:30
 * @LastEditors: lhl
 * @LastEditTime: 2024-06-21 15:49:20
 */

namespace App\Console\Commands;

use App\Models\SxTask;
use App\Services\HttpClient;
use App\Services\ShaXiang;
use App\Support\ElasticsearchSupport;
use Illuminate\Console\Command;
use Log;

class LoadVul extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'LoadVul';

    /**
     * The console command description.
     *
     * @var string
     * https://github.com/Live-Hack-CVE/full_database
     */
    protected $description = '漏洞数据入ES';

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     *
     * @return void
     */
    public function handle()
    {
        $this->info('开始执行');

        ini_set('memory_limit', '2256M');

        $handle = fopen(storage_path('hyuna_vulnclass_202406181036.json'), "rb");


        $jsonstr = "";

        $batchInsert = [];

        if ($handle) {
            while (!feof($handle)) {
                $v = fgets($handle); // 读取一行

                // echo $v;
                if (strlen($v) < 10) {
                    var_dump($v);
                }

                if (strlen($v)==3) {
                    // dd($v);
                    $jsonstr = "";
                }

                $jsonstr .= $v;

                // echo $jsonstr;
                if (strlen($v)==4) {

                    $jsonstr = rtrim($jsonstr, ",\n");

                    $jsonstr = ltrim($jsonstr, "\t");
                    
                    $arr = json_decode($jsonstr, 1);


                    if (!isset($arr['id'])) {

                        // $jsonstr = str_replace("\t", '', $jsonstr); 

                        // $newArr=json_decode($jsonstr,1);
                        dd($jsonstr, $arr);
                        Log::info('没有解析正确啊', ['str' => $jsonstr, 'arr' => $arr]);
                        continue;
                    }
                    // dd($jsonstr,$arr);
                    $batchInsert[] = $arr;
                    $jsonstr = "";
                }


                if (count($batchInsert) >= 10) {
                    $this->save($batchInsert);
                    $batchInsert=[];
                }
            }

            fclose($handle);
        }

        $this->info('执行结束');
    }


    protected function  save($data)
    {

        $index = 'vul';

        $esInit = (new ElasticsearchSupport())->initFun($index, [config('es.wxqb')]);

        $batchInsert = [];

        foreach ($data as $one) {

            $batchInsert[]['create'] = [
                '_index' => $index,
                '_id' => $one['id'],
            ];

            $one['createtime'] = time();

            $batchInsert[] = $one;
        }

        $res = $esInit->getClient()->bulk([
            'index' => $esInit->getIndex(),
            'type' => 'doc',
            'body' => $batchInsert
        ]);

        var_dump($res);
    }
}