>首页> IT >

全球今日讯!Node.js怎么实现分片上传?方法介绍

时间:2022-07-29 21:03:58       来源:转载


(资料图片仅供参考)

Node怎么实现分片上传?下面本篇文章给大家介绍一下Node.js实现分片上传的方法,希望对大家有所帮助!

解决步骤:

文件分片,减少每次请求消耗的时间,如果某次请求失败可以单独上传,而不是从头开始

通知服务端合并文件分片

控制并发的请求数量,避免浏览器内存溢出

当因为网络或者其他原因导致某次的请求失败,我们重新发送请求

文件的分片与合并

在JavaScript中,FIle对象是" Blob "对象的子类,该对象包含一个重要的方法slice,通过该方法我们可以这样分割二进制文件:

                Document    <script src="https://cdn.bootcdn.net/ajax/libs/axios/0.24.0/axios.min.js"></script>              <script>        function SliceUpload() {            const file = document.getElementById("fileInput").files[0]            if (!file) return            // 文件分片            let size = 1024 * 50; //50KB 50KB Section size            let fileChunks = [];            let index = 0;        //Section num            for (let cur = 0; cur < file.size; cur += size) {                fileChunks.push({                    hash: index++,                    chunk: file.slice(cur, cur + size),                });            }            // 上传分片            const uploadList = fileChunks.map((item, index) => {                let formData = new FormData();                formData.append("filename", file.name);                formData.append("hash", item.hash);                formData.append("chunk", item.chunk);                return axios({                    method: "post",                    url: "/upload",                    data: formData,                });            });            await Promise.all(uploadList);            // 所有分片上传完成,通知服务器合并分片            await axios({                method: "get",                url: "/merge",                params: {                    filename: file.name,                },            });            console.log("Upload to complete");        }    </script>

并发控制

如果文件很大,这样切分的分片会很多,浏览器短时间内就会发起大量的请求,可能会导致内存耗尽,所以要进行并发控制。

这里我们结合Promise.race()方法 控制并发请求的数量,避免浏览器内存溢出。

// 加入并发控制async function SliceUpload() {    const file = document.getElementById("fileInput").files[0]    if (!file) return    // 文件分片    let size = 1024 * 50; //50KB 50KB Section size    let fileChunks = [];    let index = 0;        //Section num    for (let cur = 0; cur < file.size; cur += size) {        fileChunks.push({            hash: index++,            chunk: file.slice(cur, cur + size),        });    }    let pool = []; //Concurrent pool    let max = 3; //Maximum concurrency    for (let i = 0; i < fileChunks.length; i++) {        let item = fileChunks[i];        let formData = new FormData();        formData.append("filename", file.name);        formData.append("hash", item.hash);        formData.append("chunk", item.chunk);        // 上传分片        let task = axios({            method: "post",            url: "/upload",            data: formData,        });        task.then(() => {        // 从并发池中移除已经完成的请求        let index = pool.findIndex((t) => t === task);            pool.splice(index);        });        // 把请求放入并发池中,如果已经达到最大并发量        pool.push(task);        if (pool.length === max) {            //All requests are requested complete            await Promise.race(pool);        }    }    // 所有分片上传完成,通知服务器合并分片    await axios({        method: "get",        url: "/merge",        params: {            filename: file.name,        },    });    console.log("Upload to complete");}

使代码可复用

function SliceUpload() {    const file = document.getElementById("fileInput").files[0]    if (!file) return    // 文件分片    let size = 1024 * 50; // 分片大小设置    let fileChunks = [];    let index = 0;        // 分片序号    for (let cur = 0; cur < file.size; cur += size) {        fileChunks.push({            hash: index++,            chunk: file.slice(cur, cur + size),        });    }    const uploadFileChunks = async function(list){        if(list.length === 0){            // 所有分片上传完成,通知如无            await axios({                method: "get",                url: "/merge",                params: {                    filename: file.name                }            });            console.log("Upload to complete")            return        }        let pool = []       // 并发池        let max = 3         // 最大并发数        let finish = 0      // 完成数量        let failList = []   // 失败列表        for(let i=0;i{                // 从并发池中移除已经完成的请求                let index = pool.findIndex(t=> t===task)                pool.splice(index)            }).catch(()=>{                failList.push(item)            }).finally(()=>{                finish++                // 如果有失败的重新上传                if(finish===list.length){                    uploadFileChunks(failList)                }            })            pool.push(task)            if(pool.length === max){                await Promise.race(pool)            }        }    }    uploadFileChunks(fileChunks)}

服务端接口实现

const express = require("express")const multiparty = require("multiparty")const fs = require("fs")const path = require("path")const { Buffer } = require("buffer")// file pathconst STATIC_FILES = path.join(__dirname, "./static/files")// Temporary path to upload filesconst STATIC_TEMPORARY = path.join(__dirname, "./static/temporary")const server = express()// Static file hostingserver.use(express.static(path.join(__dirname, "./dist")))// Interface for uploading slicesserver.post("/upload", (req, res) => {    const form = new multiparty.Form();    form.parse(req, function(err, fields, files) {        let filename = fields.filename[0]        let hash = fields.hash[0]        let chunk = files.chunk[0]        let dir = `${STATIC_TEMPORARY}/${filename}`        // console.log(filename, hash, chunk)        try {            if (!fs.existsSync(dir)) fs.mkdirSync(dir)            const buffer = fs.readFileSync(chunk.path)            const ws = fs.createWriteStream(`${dir}/${hash}`)            ws.write(buffer)            ws.close()            res.send(`${filename}-${hash} Section uploaded successfully`)        } catch (error) {            console.error(error)            res.status(500).send(`${filename}-${hash} Section uploading failed`)        }    })})//Merged slice interfaceserver.get("/merge", async (req, res) => {    const { filename } = req.query    try {        let len = 0        const bufferList = fs.readdirSync(`${STATIC_TEMPORARY}/${filename}`).map((hash,index) => {            const buffer = fs.readFileSync(`${STATIC_TEMPORARY}/${filename}/${index}`)            len += buffer.length            return buffer        });        //Merge files        const buffer = Buffer.concat(bufferList, len);        const ws = fs.createWriteStream(`${STATIC_FILES}/${filename}`)        ws.write(buffer);        ws.close();        res.send(`Section merge completed`);    } catch (error) {        console.error(error);    }})server.listen(3000, _ => {    console.log("http://localhost:3000/")})

更多node相关知识,请访问:nodejs 教程!

以上就是Node.js怎么实现分片上传?方法介绍的详细内容,更多请关注php中文网其它相关文章!

关键词: 并发控制 内存溢出