大文件分片上传并发

分片上传并发"/>

大文件分片上传并发

我这边使用的是boostrap-fileimput

初始化文件上传框

$(document).ready(function () {$("#file-upload_import").fileinput({uploadUrl: "#",language: "zh", //设置语言showPreview: true,autoReplace: true,// uploadUrl: "/uact/uploadfile", //上传的地址uploadExtraData: { type: "1012-0001" },allowedFileExtensions: ["json"], //接收的文件后缀,allowedPreviewTypes: ["image"], //可以预览的文件类型// initialPreviewAsData: File,// maxFileSize:10485760, //10GshowUpload: false,maxFileCount: 1,minFileCount: 1,previewTemplates: {generic: '<div class="file-preview-frame" id="{previewId}" data-fileindex="{fileindex}" data-template="{template}">\n' +'</div>\n'},dropZoneTitle: "请选择一个json文件上传",// validateInitialCount: true,// showCaption: true,fileActionSettings: {// 在预览窗口中为新选择的文件缩略图设置文件操作的对象配置showRemove: true, // 显示删除按钮showUpload: false, // 显示上传按钮showDownload: false, // 显示下载按钮showZoom: false, // 显示预览按钮showDrag: false, // 显示拖拽},});})

分片和重传

var succeed = 0;
var fail_parts = []async function progress(percent, $element) {var progressBarWidth = percent * $element.width() / 100;$element.find('div').animate({ width: progressBarWidth }, 500).html(percent + "% ");
}function updateProgress(percentage) {$('.progress .progress-bar').attr('data-transitiongoal', percentage).progressbar({ display_text: 'fill' });
}function generateUUID() {// 生成全局uuidvar d = new Date().getTime();if (window.performance && typeof window.performance.now === "function") {d += performance.now();}var uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {var r = (d + Math.random() * 16) % 16 | 0;d = Math.floor(d / 16);return (c == "x" ? r : (r & 0x3) | 0x8).toString(16);},)return uuid;
}const form = document.getElementById('upload-form');
const fileInput = document.getElementById('file-upload_import');
form.addEventListener('submit', async (e) => {e.preventDefault();$("#upload").attr("disabled", "disabled");const file = fileInput.files[0];if (!file) {alert("至少选择一个文件")return}const chunkSize = 10 * 1024 * 1024; // 10MB 分片大小// const chunkSize = 1 * 1024; // 1KB 分片大小let upload_id = generateUUID()// 切割文件const chunks = [];let offset = 0;while (offset < file.size) {const chunk = file.slice(offset, offset + chunkSize);chunks.push(chunk);offset += chunkSize;}$('.progress .progress-bar').attr('data-transitiongoal', 0).progressbar({ display_text: 'fill' });// 逐个上传分片const urls = [];for (let i = 0; i < chunks.length; i++) {const formData = new FormData();formData.append('chunk', chunks[i]);formData.append("upload_id", upload_id)formData.append('index', i);formData.append('size', file.size)urls.push({url: "你的分片上传地址",param: formData})}try {concurRequest(urls, 6).then(ret => {setTimeout(() => {$(document).trigger('chunksUploaded', [fail_parts, chunks, upload_id]);}, 1000);})} catch (error) {console.log("分片上传错误:" + error.data[0].index)fail_parts.push(error.data[0]);}
});$(document).on('chunksUploaded', async function (event, fail_parts, chunks, upload_id) {// 重传失败分片if (fail_parts.length > 0) {let retry_urls = [];for (let ele of fail_parts) {const formData = new FormData();formData.append('chunk', chunks[ele.index]);formData.append("upload_id", ele.upload_id)formData.append('retry', true)formData.append('index', ele.index)retry_urls.push({url: "你的分片上传地址",param: formData})}try {concurRequest(retry_urls, 3).then(ret => {console.log("重传分片成功！",res);})} catch (error) {console.log("重传分片" + error.msg + ":" + part.index)}}setTimeout(() => {$("#upload").removeAttr("disabled");if (fail_parts.length == 0) {const formData = new FormData();formData.append('total_chunks', chunks.length);formData.append("upload_id", upload_id);$.ajax({url: '/mongodb/import/merge/',method: 'POST',data: formData,processData: false,contentType: false,success: function (data) {if (data.status === 0) {alert('文件上传成功！');}}})} else {alert('文件上传失败,请稍后重新上传');}}, 2000);})

并发方法

/**
* 并发请求
* @param {string[]} urls 待请求的url数组
* @param {number} maxNum 最大的并发数
*/
function concurRequest(urls, maxNum) {return new Promise(resolve => {if (urls.length === 0) {/* 先考虑边界问题 */resolve([]);return;}const results = [] //存储请求结果并返回let index = 0; //下一个请求的url下标let count = 0; //计算请求的数量async function request() {if (index === urls.length) {return;}const i = index;const url = urls[index].url;const res = urls[index].param;index++try {let ret = await fetch(url, {method: "POST",body: res,});results[i] = ret //不能用push，因为这样就会出现异步完成优先写入不符合要求。var percent = ((index / urls.length).toFixed(2)) * 100;updateProgress(percent);} catch (error) {results[i] = err} finally {count++if (count === urls.length) {resolve(results);}request();}}const times = Math.min(maxNum, urls.length);for (let i = 0; i < times; i++) {request();}})
}