基于浏览器的并发请求分段上传百MB文件,加MD5片段验证和断点续传

最新推荐文章于 2024-06-14 10:07:54 发布

运维开发

最新推荐文章于 2024-06-14 10:07:54 发布

阅读量3.2k

点赞数 1

分类专栏： web前端文章标签： javascript 并发 md5 32位 64位

本文链接：https://blog.csdn.net/u013715754/article/details/51611959

版权

web前端专栏收录该内容

1 篇文章 0 订阅

订阅专栏

此功能也是耗费了大致2周的时间,开发阶段遇到了不少问题.参与这个功能的实现主要包括前端1名以及本人.

当然我们技术部门老大(经理)负责引导和提供开发中遇到的问题思路.

大致实现逻辑我在这里做一个简单的总结(本人语言组织能力不足,阅读时遇到问题和疑问,谢谢指出):

环境要求:

1.apache/nginx作为服务器.

2.PHP环境(v5.6及以上).

3.浏览器(谷歌/火狐).

实现思路:

1.将用户选择的文件进行按照固定大小进行分片(每一片的字节尽量的小).

2.使用循环每次发送定量的请求,附带当前分片的MD5密文以及编号(请求之间不等待相互的响应,发送的http请求数取决客户端浏览器内核和系统是32位还是64位).

3.后端(PHP)负责接收每次的数据并保存一个临时的文件.(每次请求对当前的片进行验证,失败返回,前端收到请求的状态再一次的发送当前请求)

4.最终将缓存的文件进行合并.(这次也需要验证,失败返回,重新上传).

代码:

=================javascript代码部分======================

<!doctype html>
<html>
<head>
<title>浏览器上传100MB文件,MD5验证,分片上传</title>
</head>
<body>
<div class="uploadMusic">
    <span>选择文件</span>
    <input type="file" id="upMusic" accept=".wav" />
</div>
<div class="uploadStart">
    <div class="progress2">
        <div class="progress-bar2">
            <div class="percent"></div>
            <a class="upload-close" href="javascript:void(0)" id="uploadClose">x</a>
        </div>
    </div>
    <div class="upRemind"></div>
</div>
<div style="top: 20px;" class="relative">
    <p id="upload_audio_html">请上传<B>20M</B>以上<B>WAV</B>格式的音乐文件</p>
    <input id="file_path" type="hidden" />
</div>
<script src="jquery-1.11.1.min.js"></script>
<script src="md5.js"></script>
<script type="application/javascript">
    $(function(){
        var preUploadSize = 1024*128;   //每次上传文件的一段，单位：字节
        var upLast=upStart=upEnd=lastStart=lastEnd=0;
        var percent=0;
        var lastFlag=1;
        var upNum,file,upNum1;//upNum每次还剩余的上传次数   upNum1总共需要上传的次数，用来获取最后一次
        var fixNum=3;      //固定的循环次数
        var uuid;          //改变文件时生成的uuid码
        var everyAddNum=3; //每次加3用来判断成功了几次
        var everyLoad=0;   //记录每次加载
        var fileAllMd5;
        var onceTime=0;
        var savePath;
        var alertFlag=false;
        var networkFlag=false;
        var fileName;
        var closeFlag=true;
        $("#uploadClose").on("click",function(){
            closeFlag=false;
            $('#upMusic')[0].disabled=false;
            $(".uploadMusic span").removeClass("file-disable");
            fileAllMd5=null;
            $('#upMusic').val("");
        })
        $('#upMusic').change(function() {
            if(this.files){
                if(onceTime<5){ //上传次数限制
                    $('#upMusic')[0].disabled=true;
                    $(".uploadMusic span").addClass("file-disable");
                    closeFlag=true;
                    var files = this.files;
                    file = files[0];
                    if(file.name){
                        fileName=file.name;
                        var arr = [];
                        var hexDigits = "0123456789abcdef";
                        for (var i = 0; i < 36; i++) {
                            arr[i] = hexDigits.substr(Math.floor(Math.random() * 0x10), 1);
                        }
                        arr[14] = "4";
                        arr[19] = hexDigits.substr((arr[19] & 0x3) | 0x8, 1);
                        uuid = arr.join("");
                        $(".upRemind").html("等待上传...");
                        $(".percent").html("0%");
                        $(".progress2").show();
                        $(".progress-bar2").width(0);
                        upLast=upStart=upEnd=lastStart=lastEnd=everyLoad=0;
                        lastFlag=1;
                        everyAddNum=3;
                        alertFlag=false;
                        networkFlag=false;
                        fileAllMd5=null;
                        var f=new FileReader();
                        f.readAsBinaryString(file);
                        $(f).load(function(e){
                            fileAllMd5=e.target.result;
                            fileAllMd5=hex_md5(fileAllMd5);
                        })
                        var size = file.size;
                        upNum=upNum1=Math.ceil(size/preUploadSize);  //21   4.1
                        start(upNum,uuid);
                    }else{
                        alert("您的上传次数过多，如有任何问题请与运营人员联系。");
                        return false;
                    }
                    onceTime++;
                }
            }else{
                alert("为获得最佳上传体验，我们推荐您使用最新版Chrome、火狐或360极速浏览器。");
            }
        })
        function start(upNum,uuid){
            if(closeFlag){
                if(upNum>fixNum){
                    for(i=0;i<fixNum;i++){
                        upStart=upEnd;
                        upEnd = upStart + preUploadSize;
                        everyLoad++;   //3 6
                        upload(file,everyLoad,lastFlag,upStart,upEnd,uuid);
                    }
                }else{
                    for(i=0;i<upNum;i++){
                        upStart=upEnd;
                        upEnd = upStart + preUploadSize;
                        everyLoad++;   //7 8
                        if(everyLoad==upNum1){
                            lastStart = upStart;
                            lastEnd = upEnd;
                            // upload(file,upNum1,flag,lastStart,lastEnd,uuid);
                        }else{
                            upload(file,everyLoad,lastFlag,upStart,upEnd,uuid);
                        }
                    }
                }
            }else{
                $(".upRemind").html("");
                $(".progress2").hide();
                return false;
            }
        }
        var again=function(data){
            var uploadMusicUrl=$("#uploadMusicUrl").val();
            $.ajax({
                type: 'POST',
                url: uploadMusicUrl,
                data: data,
                timeout : 60000, //超时时间设置30秒请求
                contentType: false,
                // 避开jQuery对 data 对象的默认处理
                processData: false,
                crossDomain:true,
                success:function(msg){
                    var msg=$.parseJSON(msg);
                    if(msg.state){
                        upLast++;
                        showprogress(upLast*preUploadSize);
                        if(upLast==everyAddNum){
                            upNum-=fixNum;
                            if(upNum>fixNum){
                                everyAddNum+=3;
                                start(upNum,uuid);
                            }else{
                                upNum=upNum;
                                start(upNum,uuid);
                            }
                        }
                        if(upLast==upNum1-1){
                            lastFlag=2;
                            upload(file,upNum1,lastFlag,lastStart,lastEnd,uuid);
                        }
                        savePath=msg.savePath;
                        $('#file_path').val(msg.savePath);
                    }else{
                        if(msg.errorState=="fragmentFail"){
                            upLast--;
                            upload(file,msg.n,msg.start,msg.end,uuid);
                        }else if(msg.errorState!="methodFail"){
                            if(!alertFlag){
                                alertFlag=true;
                                alert(""+msg.msg+"，请重新上传");
                            }
                            $(".progress2").hide();
                            $(".upRemind").html("");
                            $('#upMusic')[0].disabled=false;
                            $(".uploadMusic span").removeClass("file-disable");
                            return false;
                        }
                    }
                },
                error: function (jqXHR, textStatus, errorThrown) {
                    if(jqXHR.statusText == 'timeout' || textStatus == 'timeout' || errorThrown == 'timeout')
                    {
                        if(!networkFlag){
                            networkFlag=true;
                            alert("您当前的网络不佳，请稍后再试");
                        }
                        $(".upRemind").html("上传失败").css("color","#f00");
                        $(".progress2").hide();
                        return false;
                    }

                }
            })
        }
        function upload(file,i,lastFlag,start,end,uuid) {
            var data = new FormData();
            var part=file.slice(start,end);
            var r=new FileReader();
            r.readAsBinaryString(part);
            $(r).load(function(e){
                if(lastFlag==2){
                    //fileAllMd5=hex_md5(file);
                    data.append("fileAllMd5",fileAllMd5);
                }
                var bolb=e.target.result;
                var b=hex_md5(bolb);
                data.append("name", encodeURIComponent(file.name));
                data.append("n",i);
                data.append("file", file.slice(start, end));
                data.append("md5",b);
                data.append("flag",lastFlag);
                data.append("start",start);
                data.append("end",end);
                data.append("uuid",uuid);
                data.append('mirotime',new Date().getTime());
                again(data);
            })
        }

        function showprogress(per) {
            if(per<file.size){
                percent=(per/file.size)*100;
                $(".upRemind").html("上传中");
            }else{
                percent=100;
                $(".upRemind").html(""+fileName+"上传完成");
                $('.progress2').hide();
                $('#upMusic')[0].disabled=false;
                $(".uploadMusic span").removeClass("file-disable");
            }
            //console.log(percent);
            $('.progress-bar2').width(percent+ '%');
            $(".percent").html(Math.ceil(percent)+ '%');
        }
    })
</script>
</body>
</html>

===================php代码部分========================

<?php
date_default_timezone_set('PRC');
error_reporting(E_ALL & ~E_NOTICE);
$state = ['state' => false];

$state['errorState'] = 'methodFail';//传输方式
if(empty($_POST)):
    $state['msg'] = '上传方式仅限POST提交';
    $state['msgInfo'] = '传输的方法有误，仅限POST方式提交';
    echo json_encode($state);
    exit;
endif;
$oldFileName =  $_POST['name'] ? urldecode(trim($_POST['name'])) : null; //原始文件名
$md5Name = md5(substr($oldFileName,strpos($oldFileName,'.wav'))).'.wav'; //文件名
$uid = isset($_POST['uid']) ? (int)$_POST['uid'] : null; //uid
$uuid = isset($_POST['uuid']) ? htmlspecialchars(trim($_POST['uuid'])) : null; //唯一的uuid
$md5 = isset($_POST['md5']) ? htmlspecialchars(trim($_POST['md5'])) : null; //文件加密的字符串
$fileAllMd5 = isset($_POST['fileAllMd5']) ? htmlspecialchars(trim($_POST['fileAllMd5'])) : null; //整体文件加密的字符串
$flag = is_numeric($_POST['flag']) ? (int)$_POST['flag'] : null;//1 不是最后一次传输 2是最后一次传输
$n = is_numeric($_POST['n']) ? (int)$_POST['n'] : null;
$start = is_numeric($_POST['start']) ? (int)$_POST['start'] : null; //起始的位置
$end = is_numeric($_POST['end']) ? (int)$_POST['end'] : null; //截止的位置


$state['errorState'] = 'paramsFail';//起始的错误代号
$state['msg'] = '传输的参数有误';
if(!is_numeric($start) || !is_numeric($end) || !is_numeric($n) ):
    $state['msgInfo'] = '传输的参数start或end或n必须是数字';
    echo json_encode($state);
    exit;
endif;
if($flag != 1 && $flag != 2):
    $state['msgInfo'] = '参数有误flag必须是1或2';
    echo json_encode($state);
    exit;
endif;
if($md5 === null || $oldFileName === null || $uuid === null):
    $state['msgInfo'] = '缺少必须的参数md5或文件名或uuid';
    echo json_encode($state);
    exit;
endif;
if(strpos($md5,'.') !== false):
    $state['msgInfo'] = '参数有误，md5校验码传输不符合要求';
    echo json_encode($state);
    exit;
endif;
if(strpos($uuid,'.')):
    $state['msgInfo'] = '参数有误，uuid传输有误';
    echo json_encode($state);
    exit;
endif;

$path_parts = pathinfo($oldFileName);
$ext = strtolower($path_parts['extension']);
$extArray = ['wav'];
if(!in_array($ext,$extArray)):
    $state['msgInfo'] = '文件后缀必须是.wav文件';
    echo json_encode($state);
    exit;
endif;

$state['errorState'] = 'fileUploadFail';//文件上传出错
$state['msg'] = '文件上传出错';
switch ($_FILES['file']['error']):
    case UPLOAD_ERR_OK:
        $msgInfo = '';
        break;
    case UPLOAD_ERR_INI_SIZE:
    case UPLOAD_ERR_FORM_SIZE:
        $msgInfo = "上传文件超出限制";
        break;
    case UPLOAD_ERR_PARTIAL:
        $msgInfo = "文件上传不完整";
        break;
    case UPLOAD_ERR_NO_FILE:
        $msgInfo = "没有文件被上传";
        break;
    case UPLOAD_ERR_NO_TMP_DIR:
        $msgInfo = "文件不能被缓存";
        break;
    case UPLOAD_ERR_CANT_WRITE:
        $msgInfo = "文件写入失败";
        break;
    case UPLOAD_ERR_EXTENSION:
        $msgInfo = "File upload stopped by extension";
        break;
    default:
        $msgInfo = "Unknown upload error";
        break;
endswitch;
if($msgInfo):
    $state['msgInfo'] = $msgInfo;
    echo json_encode($state);
    exit;
endif;
if(!is_uploaded_file($_FILES['file']['tmp_name'])):
    $state['msgInfo'] = '上传文件必须是post上传';
    echo json_encode($state);
    exit;
endif;

$state['errorState'] = 'fileCreateFail';//权限错误
$state['msg'] = '权限错误';
//默认上传只需要存放在缓存目录，完成之后需要将文件移动
$runTimePath =  '/mnt/www/rest/runtime/cache/tmp/'.$uuid;//缓存目录
if (!file_exists($runTimePath)):
    if(!mkdir($runTimePath,0777,true)):
        $state['msgInfo'] = '缓存目录创建失败';
        echo json_encode($state);
        exit;
    endif;
endif;

$rootPath =  '/mnt/www/sns/data/uploads/';
$savePath = 'original/music/' . date('Y');
if (!file_exists($rootPath.$savePath)):
    if(!mkdir($rootPath.$savePath,0777,true)):
        $state['msgInfo'] = '文件保存目录创建失败';
        echo json_encode($state);
        exit;
    endif;
endif;


$state['errorState'] = 'fragmentFail';//文件单独片验证失败
$state['msg'] = '文件单片验证失败';
//以上是文件发送请求基本的参数验证
if(md5_file($_FILES['file']['tmp_name']) != $md5):
    $state['n'] = $n;
    $state['start'] = $start;
    $state['end'] = $end;
    $state['msgInfo'] = '检验文件失败';
    echo json_encode($state);
    exit;
endif;


$runtimeFileName = $runTimePath.'/'.$md5.'_'.$n;
$state['msg'] = '';
$state['errorState'] = '';
if(!file_exists($runtimeFileName)):
    @move_uploaded_file($_FILES['file']['tmp_name'],$runtimeFileName);
    $state['state'] = true;
else:
    $state['state'] = true;
endif;
//最后一次请求的确认条件
if($flag == 2 && $fileAllMd5 != null):
    $fileSortArr = [];
    if(is_dir($runTimePath)):
        if($dh = @opendir($runTimePath)):
            while (($file = @readdir($dh)) !== false):
                if(is_file($runTimePath.'/'.$file) && strpos($file,'_') !== false):
                    $tmp = explode('_',$file);
                    $fileSortArr [$tmp[1]] = $file;
                endif;
            endwhile;
            ksort($fileSortArr);//排序需要保持下标索引的值
            $savePath .= '/' . $uid . '_' . strtotime(date('Y-m-dHis')) . '.' . $ext;
            $saveFileFp = @fopen($runTimePath.'/'.$md5Name,'wb');//打开即将保存的文件
            if(!$saveFileFp):
                $state['errorState'] = 'fileCreateFail';//权限错误
                $state['msg'] = '权限错误';
                $state['msgInfo'] = '文件句柄打开失败';
                echo json_encode($state);
                exit;
            endif;
            foreach ($fileSortArr as $key => $value):
                $fp = @fopen($runTimePath.'/'.$value, "rb");
                if(!$fp):
                    $state['errorState'] = 'fileCreateFail';//权限错误
                    $state['msg'] = '权限错误';
                    $state['msgInfo'] = '文件句柄打开失败';
                    echo json_encode($state);
                    exit;
                endif;
                if (@flock($fp, LOCK_EX)): // 进行排它型锁定
                    $cont = @fread($fp, filesize($runTimePath.'/'.$value));
                    if(!@fwrite($saveFileFp,$cont)):
                        $state['errorState'] = 'fileCreateFail';//权限错误
                        $state['msg'] = '权限错误';
                        $state['msgInfo'] = '文件拼接错误';
                        echo json_encode($state);
                        exit;
                    endif;
                    @flock($fp, LOCK_UN);    //释放锁定
                else:
                    $state['errorState'] = 'fileCreateFail';//权限错误
                    $state['msg'] = '权限错误';
                    $state['msgInfo'] = "文件锁定失败";
                    echo json_encode($state);
                    exit;
                endif;
                @fclose($fp);
            endforeach;
            unset($value);
            @fclose($saveFileFp);
            @closedir($dh);
            //判断合并之后的文件是否异常
            $phpFileMd5Str = md5_file($runTimePath.'/'.$md5Name);
            if($phpFileMd5Str != $fileAllMd5):
                $state['state'] = false;
                $state['errorState'] = 'fileVerifyFail';//文件校验失败
                $state['msg'] = '文件校验失败';
                $state['msgInfo'] = '文件上传受损';
                $state['phpFileMd5Str'] = $phpFileMd5Str;
                $state['fileAllMd5'] = $fileAllMd5;
                echo json_encode($state);
                exit;
            endif;
            if(!rename($runTimePath.'/'.$md5Name,$rootPath.$savePath)):
                $state['state'] = false;
                $state['errorState'] = 'moveFail';//文件移动失败
                $state['msg'] = '文件移动失败';
                $state['msgInfo'] = '文件移动失败';
                echo json_encode($state);
                exit;
            endif;
            $state['state'] = true;
            $state['savePath'] = $savePath;
        else:
            $state['errorState'] = 'fileCreateFail';//权限错误
            $state['msg'] = '权限错误';
            $state['msgInfo'] = '目录打开失败，请重新尝试';
        endif;
    else:
        $state['errorState'] = 'fileCreateFail';//权限错误
        $state['msg'] = '权限错误';
        $state['msgInfo'] = '文件目录异常，请重新尝试';
    endif;
endif;
echo json_encode($state);
exit;

以上是源码部分,仅仅包含简单的注释.

运维开发

关注

1
点赞
踩
2

收藏

觉得还不错? 一键收藏
0
评论
基于浏览器的并发请求分段上传百MB文件,加MD5片段验证和断点续传

此功能也是耗费了大致2周的时间,开发阶段遇到了不少问题.参与这个功能的实现主要包括前端1名以及本人.当然我们技术部门老大(经理)负责引导和提供开发中遇到的问题思路.大致实现逻辑我在这里做一个简单的总结(本人语言组织能力不足,阅读时遇到问题和疑问,谢谢指出):环境要求:1.apache/nginx作为服务器.2.PHP环境(v5.6及以上).3.浏览器(谷歌/火狐).
复制链接

扫一扫

专栏目录