I am trying to create the blocks and commit them in the same code using the below logic:
PutStream.js
const request = require('request')
const fs = require('fs')
const account = 'Enter your container name here'
const containerName = 'stream-test-container'
const blobName = 'myBlob.txt'
const strTime = new Date().toUTCString()
const _ = require('lodash')
let i = 0
const blockIdArray = []
const token = 'Enter Your Token Here'
const readStream = fs.createReadStream('smallTextFile.txt', { highWaterMark: 50 })
readStream.on('data', function (chunk) {
i++
const blobContent = chunk
const contentLength = chunk.length
const blockId = Buffer.from('block' + i).toString('base64')
blockIdArray.push(blockId)
const optionsPutBlock = {
url: `https://${account}.blob.core.windows.net/${containerName}/${blobName}?comp=block&blockid=${blockId}`,
headers: {
Authorization:
`Bearer ${token}`,
'x-ms-date': strTime,
'x-ms-version': '2019-02-02',
'Content-Length': contentLength
},
body: blobContent
}
request.put(optionsPutBlock, callbackPutBlock)
})
readStream.on('end', () => {
const xmlBlockList = '<?xml version="1.0" encoding="utf-8"?><BlockList>' + (_.map(blockIdArray, (id) => { return `<Latest>${id}</Latest>` })).join('') + '</BlockList>'
console.log(xmlBlockList)
const xmlBlockListLength = new TextEncoder().encode(xmlBlockList).length
const optionsPutBlockList = {
url: `https://${account}.blob.core.windows.net/${containerName}/${blobName}?comp=blocklist`,
headers: {
Authorization:
`Bearer ${token}`,
'x-ms-date': strTime,
'x-ms-version': '2019-02-02',
'Content-Length': xmlBlockListLength,
'Content-Type': 'application/text-plain'
},
body: xmlBlockList
}
console.log(optionsPutBlockList.body)
request.put(optionsPutBlockList, callbackPutBlockList)
})
function callbackPutBlock (error, response, body) {
console.log(response.statusCode, response.statusMessage, Block created)
if (error) {
console.log(error)
}
}
function callbackPutBlockList (error, response, body) {
console.log(response.statusCode, response.statusMessage)
if (error) {
console.log(error)
}
}
The contents of smallTextFile.txt
are:
This is a small file. Hello from Nodejs. How are you?
This is line number 2. Dumm text Lorem
This is line number 3 of the dummy text
This is line number 4 four.
Output of PutStream.js
:
<?xml version="1.0" encoding="utf-8"?><BlockList><Latest>YmxvY2sx</Latest><Latest>YmxvY2sy</Latest><Latest>YmxvY2sz</Latest><Latest>YmxvY2s0</Latest></BlockList>
201 Created Block created
400 The specified block list is invalid.
201 Created Block created
201 Created Block created
201 Created Block created
When I try to run this it doesn't commit the blocks. I came to know this after I ran Get Block List
after the above code PutStream.js
, As all the blocks were in the uncommitted list.
Can anyone tell why does above logic doesn't readily commit blocks after creating them?
References:
Issue committing the blocks : Above code is sending PutBlockList requests even before the Blocks are created by PutBlock due to which it is failing.
Solution : The problem committing blocks got resolved by increasing the size of file to 1Mb and high watermark to 1024*256 and by making sure that PutBlockList is called only after all the blocks have been uploaded . Thanks to @mayank patel.
Note: keep length of the block ids same as issue may arise when the number of blocks become 10 or more ,as suggested by @gaurav mantri