Node.js中大文件上传至Azure blob的处理

Question

我的目标是将一个大文件（任何文件类型）上传到Azure Blob中。

1. Convert file into stream and upload into Azure blob
2. Convert file into block streams and commit all the blocks to Azure blob in parallel

    var blobName = "azure-blob-name.doc";
    var fileSize = fs.statSync(fileName).size;
    var readableStream = fs.createReadStream(fileName, 'utf-8');
        blobService.createBlockBlobFromStream(containerName, blobName, readableStream, fileSize, function (error, result, response) { 
            if(error) console.log(error);
            console.log({message : "blob uploaded"});
        });

var fileName = "local-file-path/filename.doc";
    var blobName = "azure-blob-name.doc";
    var fileSize = fs.statSync(fileName).size;
    var readableStream = fs.createReadStream(fileName, 'utf-8');
    var bl=1;
    var blockIds = [];
    readableStream.on("data", function (chunk) {
        var blockId = 'block'+bl;
        bl++;
        blobService.createBlockFromStream(blockId, containerName, blobName, chunk, chunk.length, function(error, response){
            if(error) console.log(error);

            blockIds.push(blockId);
                    console.log({message : "block created"});
        });
    });

下面是我得到的错误

    D:\main-workspace\file-management\node_modules\azure-storage\lib\common\services\storageserviceclient.js:522
                body.outputStream.on('open', function () {
                                  ^
    TypeError: body.outputStream.on is not a function

1. 方法1工作很好，但上传时间太长。（统计数据：上传40 MB文件花了80秒）。
2. 我的第二个方法是正确的一个提高性能？如果是，如何实现此方法？

有没有更好的方法来提高性能？在这方面有什么建议吗？

      const stream = require('stream');
      var fileName = "local-file-path/filename.doc";
      var blobName = "azure-blob-name.doc";
      var fileSize = fs.statSync(blobName).size;
      var customBlockSize = 1200; //1.2 MB CAP           
      var NoOfBlocks = Math.ceil(fileSize/customBlockSize);
      var blockIdLength = NoOfBlocks.toString().length;
      var readableStream = fs.createReadStream(fileName, 'utf-8');
      var bl=1;
      var blockIds = [];
      readableStream.on("data", function (chunk) {         
          var blockId = 'block'+ blockGuId(bl, blockIdLength);//Block ids must be of same length
          bl++;
          const bufferStream = new stream.PassThrough({
            highWaterMark: chunk.length
          });
          bufferStream.end(chunk);
          blobService.createBlockFromStream(blockId, containerName, blobName, bufferStream, chunk.length, function(error, response){
              if(error) console.log(error);

              blockIds.push(blockId);
              console.log({message : "block ("+blockId+") created"});
          });
      });

      readableStream.on("end", function(){

        console.log('readableStream.on "end"');

        blobService.commitBlocks(containerName, blobName, blockIds, function(error, result){
          if(error) console.log(error);

          console.log({message : "all blocks uploaded"});
          var hrend = process.hrtime(hrstart)
          console.info('Execution time (sec ms): %ds %dms', hrend[0], hrend[1] / 1000000)
        });
      });

      blockGuId(number, blockIdLength)
      {
        return Array(Math.max(blockIdLength - String(number).length + 1, 0)).join(0) + number;
      }

下面是套接字错误

{ Error: socket hang up at createHangUpError (_http_client.js:323:15) at TLSSocket.socketCloseListener (_http_client.js:364:25) at TLSSocket.emit (events.js:187:15) at _handle.close (net.js:610:12) at TCP.done (_tls_wrap.js:386:7) code: 'ECONNRESET' }

我在这里错过了什么？为什么在创建块时发生套接字错误？

Answer 1

您能否尝试以下代码：

const stream = require('stream');

var fileName = "local-file-path/filename.doc";
    var blobName = "azure-blob-name.doc";
    var fileSize = fs.statSync(fileName).size;
    var readableStream = fs.createReadStream(fileName, 'utf-8');
    var bl=1;
    var blockIds = [];
    readableStream.on("data", function (chunk) {
        //var blockId = 'block'+bl;
        bl++;
        var blockId = UseGuidHere();//Block ids must be of same length
        const bufferStream = new stream.PassThrough({
          highWaterMark: chunk.length
        });
        bufferStream.end(chunk);
        blobService.createBlockFromStream(blockId, containerName, blobName, bufferStream, chunk.length, function(error, response){
            if(error) console.log(error);

            blockIds.push(blockId);
                    console.log({message : "block created"});
        });
    });

正如代码中提到的，请为您的块id分配一个GUID，因为块id必须具有相同的长度。在您当前的逻辑中，在9个块之后，您的块id的长度将会改变，这将导致上传过程中的失败。