Thinkphp使用redis消息队列将excel文件数据分批导入到数据库

在index.html中添加导入按钮

对应的JS文件中添加导入请求地址

控制器重写导入类

public function import()
    {
        //return parent::import();
        $file = $this->request->request('file');
        if (!$file) {
            $this->error(__('Parameter %s can not be empty', 'file'));
        }
        $filePath = ROOT_PATH . DS . 'public' . DS . $file;
        if (!is_file($filePath)) {
            $this->error(__('No results were found'));
        }
        //实例化reader
        $ext = pathinfo($filePath, PATHINFO_EXTENSION);
        if (!in_array($ext, ['csv', 'xls', 'xlsx'])) {
            $this->error(__('Unknown data format'));
        }
        if ($ext === 'csv') {
            $file = fopen($filePath, 'r');
            $filePath = tempnam(sys_get_temp_dir(), 'import_csv');
            $fp = fopen($filePath, "w");
            $n = 0;
            while ($line = fgets($file)) {
                $line = rtrim($line, "\n\r\0");
                $encoding = mb_detect_encoding($line, ['utf-8', 'gbk', 'latin1', 'big5']);
                if ($encoding != 'utf-8') {
                    $line = mb_convert_encoding($line, 'utf-8', $encoding);
                }
                if ($n == 0 || preg_match('/^".*"$/', $line)) {
                    fwrite($fp, $line . "\n");
                } else {
                    fwrite($fp, '"' . str_replace(['"', ','], ['""', '","'], $line) . "\"\n");
                }
                $n++;
            }
            fclose($file) || fclose($fp);
            $reader = new Csv();
        } elseif ($ext === 'xls') {
            $reader = new Xls();
        } else {
            $reader = new Xlsx();
        }

        //导入文件首行类型,默认是注释,如果需要使用字段名称请使用name
        $importHeadType = isset($this->importHeadType) ? $this->importHeadType : 'comment';

        $table = $this->model->getQuery()->getTable();
        $database = \think\Config::get('database.database');
        $fieldArr = [];
        $list = db()->query("SELECT COLUMN_NAME,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA = ?", [$table, $database]);
        foreach ($list as $k => $v) {
            if ($importHeadType == 'comment') {
                $v['COLUMN_COMMENT'] = explode(':', $v['COLUMN_COMMENT'])[0]; //字段备注有:时截取
                $fieldArr[$v['COLUMN_COMMENT']] = $v['COLUMN_NAME'];
            } else {
                $fieldArr[$v['COLUMN_NAME']] = $v['COLUMN_NAME'];
            }
        }

        //加载文件
        $insert = [];
        try {
            if (!$PHPExcel = $reader->load($filePath)) {
                $this->error(__('Unknown data format'));
            }
            $currentSheet = $PHPExcel->getSheet(0);  //读取文件中的第一个工作表
            $allColumn = $currentSheet->getHighestDataColumn(); //取得最大的列号
            $allRow = $currentSheet->getHighestRow(); //取得一共有多少行
            $maxColumnNumber = Coordinate::columnIndexFromString($allColumn);
            $fields = [];
            for ($currentRow = 1; $currentRow <= 1; $currentRow++) {
                for ($currentColumn = 1; $currentColumn <= $maxColumnNumber; $currentColumn++) {
                    $val = $currentSheet->getCellByColumnAndRow($currentColumn, $currentRow)->getValue();
                    $fields[] = $val;
                }
            }

            $one_row = config('site.excel_read_row'); //单次读取数据行数
            $all_time = ceil($allRow / $one_row); //读取数据总次数
            $start_row = 2;
            $number = date('Ymd',time());
            for ($read_times = 1; $read_times <= $all_time; $read_times++) {
                $start =  $start_row + ($read_times - 1) * $one_row ;
                for ($currentRow = $start; $currentRow <= $one_row * $read_times; $currentRow++) {
                    $values = [];
                    for ($currentColumn = 1; $currentColumn <= $maxColumnNumber; $currentColumn++) {
                        $val = $currentSheet->getCellByColumnAndRow($currentColumn, $currentRow)->getCalculatedValue();
                        $values[] = is_null($val) ? '' : $val;
                    }
                    $row = [];
                    $temp = array_combine($fields, $values);
                    foreach ($temp as $k => $v) {
                        if (isset($fieldArr[$k]) && $k !== '') {
                            if($k == '时间') {
                                $row[$fieldArr[$k]] = ($v == '') ? '' : date('Y-m-d H:i',(intval($v) - 25569) * 86400);//处理excel中时间
                            }else{
                                $row[$fieldArr[$k]] = $v;
                            }
                        }
                    }
                    if ($row) {
                        $insert[$read_times][] = $row;
                        $number = !empty($row['number']) ? $row['number'] :  $number;
                    }
                    if ($currentRow == $allRow)
                        break;
                }
                //将文件行数分批写入队列处理
                $isPushed  = \think\Queue::push('app\job\DarenImport@importData', $insert[$read_times], $queue = null);
                if ($isPushed !== false) {
                    echo '['.$read_times.']'." 队列加入成功 \r\n";
                } else {
                    echo "队列加入失败 \r\n";
                }
                unset($insert[$read_times]);
            }

        } catch (Exception $exception) {
            $this->error($exception->getMessage());
        }
        $this->success();
    }

创建队列消费者

<?php
namespace app\job;
use think\Db;
use think\queue\Job;
use app\admin\model\Daren;

class DarenImport
{
    /**
     * 数据导入
     * @param array $data  内容
     * @return
     */
    public function importData(Job $job, array $data)
    {
        $isJobDone = $this->send($data);
        if ($isJobDone) {
            echo '导入成功！';
            //成功删除任务
            $job->delete();
        } else {
            //任务轮询2次后删除
            if ($job->attempts() > 1) {
                // 第1种处理方式：重新发布任务,该任务延迟10秒后再执行
                //$job->release(10);
                // 第2种处理方式：原任务的基础上1分钟执行一次并增加尝试次数
                //$job->failed();
                // 第3种处理方式：删除任务
                $job->delete();
            }
        }
    }

    /**
     * 根据消息中的数据进行实际的业务处理
     * @param array|mixed    $data     发布任务时自定义的数据
     * @return boolean                 任务执行的结果
     */
    private function send($data)
    {
        foreach ($data as  $k => $v) {
            //处理你要导入的数据
            $data[$k]['joint_username'] = $membser['username'] ?? ''; 
            if ( Daren::get(['mobile' => $data[$k]['mobile']]) ) {
                (new Daren())->where(['mobile' => $data[$k]['mobile']])->update($data[$k]);

            }else{
                Daren::create($data[$k]);
            }
            unset($data[$k]);
        }
        return true;

    }

}