React: Corrupted Chunked File Upload with Next.js

I'm working on the functionality of file upload (any kind of files), with axios and Next.js. I'm limiting chunk size of uploaded file to 768kB (as next.js server dev only allows up to 1MB 😁) For image and video files, it works correctly. The uploaded files have exact file size with the original ones. But for Android app (.apk), it doesn't. The uploaded APK files have smaller size with the original ones.

Frontend code: src/lib/api-helpers.js

const _=require('lodash');

const helpers={
  /**
   * Helper for uploading chunked file
   * @param {string} url 
   * @param {File} file 
   * @param {(progress:number)=>void} onUploadProgress 
   * @param {(attachment:import('models/Attachments').IAttachmentsSafe)=>void} onUploadFinish 
   * @returns void
   */
  chunkUpload(url,file,onUploadProgress,onUploadFinish)
  {
    const config=typeof url=='object'?_.extend({"chunkSize":786432},url):{
      "url":url,
      "chunkSize":786432
    };
    if(!file && config['file']) {
      file=config.file;
      delete config.file
    };
    if(!onUploadProgress && config['onUploadProgress'])
    {
      onUploadProgress=config.onUploadProgress;
      delete config.onUploadProgress;
    }
    if(!onUploadFinish && config['onUploadFinish'])
    {
      onUploadFinish=config.onUploadFinish;
      delete config.onUploadFinish;
    }

    if(!onUploadProgress)
    {
      console.error('No onUploadProgress');
      return;
    }

    if(!onUploadFinish)
    {
      console.error('No onUploadFinish');
      return;
    }
    
    var n,response;
    config.url=helpers.getFullUrl(config.url);
    const maxChunk=Math.ceil(file.size/config.chunkSize);
    const fileData={
      tempId:helpers.randHex(8),
      fileName:file.name,
      mime:file.type,
      fileSize:file.size,
      parts:maxChunk,
      index:0,
      content:""
    };

    if(config['additionalData']) _.extend(fileData,config.additionalData);

    const canceller={
      c:null,
      cancelled:false,
      cancel()
      {
        if(!this.c) return;

        this.c();
        this.cancelled=true;
      }
    }
    const CancelToken = new axios.CancelToken((c)=>{
      canceller.c=c;
    });

    const chunkRe=new RegExp('.{1,'+config.chunkSize+'}','g');

    (async function(){
      var base64File=await helpers.toBase64(file);
      base64File=base64File.replace(base64File.substr(0,base64File.search(',')+1),'');
      const base64Arr=base64File.match(chunkRe);

      for(const [i,item] of base64Arr.entries())
      {
        if(canceller.cancelled) break;
        fileData.index=i;
        fileData.content=item;
    
        try{
          response=await axios({
            url:config.url,
            method:"POST",
            data:fileData
          },{
            cancelToken:CancelToken
          });
        }catch(error){
          console.log(error);
        }finally{
          if(response.status==200 && response.data.success)
          {
            if(i==maxChunk-1) onUploadFinish(response.data.result);
            else if(typeof response.data.result.size=='number') onUploadProgress((i+1)*100/maxChunk);
          }
        }
      }
    })();

    return canceller;
  },
  /**
   * Get full path based on file system, only runs on server
   * @param {string} path 
   * @returns string
   */
  getFullPath(path)
  {
    if(path.indexOf(global.WORKSPACE_PATH)===0) return path;
    return pathJoin(global.WORKSPACE_PATH,path);  
  },
  /**
   * Get full file URL
   * @param {string} url 
   * @returns string
   */
  getFullUrl(url)
  {
    if(/^http/.test(url)||(/^\//.test(url))) return url;
  
    var baseUri;
  
    //if run at web browser
    if(typeof document=='object'?document && settings:false) baseUri=settings.baseUrl;
    //if run at server
    else if(process?process.env:false) baseUri=process.env.BASE_URI!=undefined?process.env.BASE_URI:'';

    return baseUri.replace(/[\/]+$/,'')+'/'+url.replace(/^[\/]+/,'');
  },
  randHex(size)
  {
    var maxlen = 16,
      min = Math.pow(16,Math.min(size,maxlen)-1),
      max = Math.pow(16,Math.min(size,maxlen)) - 1,
      n   = Math.floor( Math.random() * (max-min+1) ) + min,
      r   = n.toString(16);

    while ( r.length < size ) r = r + randHex( size - maxlen );

    return r;
  },
  toBase64(file)
  {
    return new Promise((resolve, reject) => {
      const reader = new FileReader();
      reader.onload = () => resolve(reader.result);
      reader.onerror = error => reject(error);
      reader.readAsDataURL(file);
    });
  }
};
module.exports=helpers;

src/models/database.js

const mongoose=require('mongoose');

var cached = global.mongo;
if (!cached) cached = global.mongo = { conn: null, promise: null };

const database={
  async function setUpDb() {
    if(cached.conn?cached.conn.client.isConnected():false) {
      return cached.conn;
    }
  
    if (!cached.promise) {
      cached.conn= await mongoose.createConnection(process.env.MONGODB_URI,{
        useNewUrlParser:true,
        useUnifiedTopology:true,
        dbName:process.env.MONGODB_DB
      });
    
      if(cached.conn.readyState!==1) throw new Error("connection not connected");
    }

    return cached.conn;
  },
  async function setupModel(modelName,modelDef,collectionName)
  {
    const conn=await database.setUpDb();

    if(conn.models[modelName]) return conn.models[modelName];

    return conn.model(modelName,modelDef,collectionName);
  }
}
module.exports=database;

src/models/Attachment.js

if(typeof window=='object'?window && document:false)
{
    module.exports={};
}else{
  const {setupModel}=require('../middlewares/database');
  const Mongoose=require('mongoose');
  const {getFullUrl}=require('../lib/api-helpers');

  const attachmentSchema=new Mongoose.Schema({
    name:{
        type:String,
        index:true,
        unique:true
    },
    fileName:String,
    filePath:String,
    fileSize:Number,
    fileUrl:String,
    mime:{
        type:String,
        index:true
    },
    attributes:Object,
    isSong:{
        type:Boolean,
        default:false,
        index:true
    },
    thumbnail:Object,
    posterPic:{
        type:Mongoose.Types.ObjectId,
        ref:'Attachments'
    }
  });

  //Covert to "safe" JS object, without filePath
  attachmentSchema.virtual('safe').get(async function()
  {
        const ret=JSON.parse(JSON.stringify(this.toObject()));
        delete ret.filePath;
        ret.fileUrlFull=getFullUrl(this.fileUrl);

        if(this.thumbnail) ret.thumbnail={
            fileName:this.thumbnail.fileName,
            fileSize:this.thumbnail.fileSize,
            fileUrlFull:getFullUrl(this.thumbnail.fileUrl),
            attributes:this.thumbnail.attributes
        };

        if(this.posterPic) {
            if(!this.populated('posterPic')) await this.populate('posterPic').execPopulate();
            ret.posterPic=await this.posterPic.safe;
        }
        
        return ret;
    }
  );

  async function asyncAttachments()
  {
    return await setupModel('Attachments',attachmentSchema,'attachments');
  };

  const AttachmentModule=module.exports=asyncAttachments;

  AttachmentModule.createFromFile=async function(fullPath,attrs)
  {
    if(!attrs) attrs={
        fileSize:0,
        mime:''
    };
    let comparePath=getFullPath('public'),karaokeChannel='left';

    const path=require('path');
    const fs=require('fs');
    const contentType=require('mime-types').contentType;

    if(fullPath.substr(0,comparePath.length)!=comparePath) fullPath=path.join(comparePath,fullPath);

    if(!fs.existsSync(fullPath)) throw ReferenceError('FILE_NOT_FOUND');

    var fileSize=0,mime='';

    if(attrs?!(fileSize=(attrs['fileSize']?attrs.fileSize:'')):true)
    {
        const stats=fs.statSync(fullPath);
        fileSize=stats.size;
    }

    if(attrs?!(mime=attrs['mime']?attrs.mime:''):true) mime=contentType(path.basename(fullPath));

    if(attrs['fileSize']!=undefined) delete attrs.fileSize;
    if(attrs['mime']!=undefined) delete attrs.mime;

    const doc={
        name:path.basename(fullPath),
        fileName:path.basename(fullPath),
        filePath:fullPath.substr(getFullPath('').length+1).replace(/\\/g,'/'),
        fileSize,
        fileUrl:"",
        mime,
        attributes:null
    };

    if(attrs['posterPic'])
    {
        doc.posterPic=attrs.posterPic;
        delete attrs.posterPic;
    }

    let names=path.basename(fullPath).split('.');
    let ext=names.pop();
    doc.name=names.join('.');
    //if Windows server
    doc.fileUrl=(/^public[\\/]/i.test(doc.filePath)?doc.filePath.substr(7):doc.filePath).replace(/\\/g,'/');

    if(/^image\//.test(attrs.mime))
    {
        const sharp=require('sharp')(fullPath);
        const metadata=await sharp.metadata();

        doc.attributes={
            width:metadata.width,
            height:metadata.height
        };
        const thumbnailSize=await (require('../lib/Settings')).getItem('thumbnailSize');

        if(doc.attributes.width>thumbnailSize.width||doc.attributes.height>thumbnailSize.height)
        {
            let scale=1;

            if(doc.attributes.width>doc.attributes.height) scale=thumbnailSize.width/doc.attributes.width;
            else scale=thumbnailSize.height/doc.attributes.height;

            const nw=doc.attributes.width*scale,nh=doc.attributes.height*scale;
            const thumbnailFile=fullPath.substr(0,fullPath.length-(ext.length+1))+`-${nw}x${nh}.${ext}`;

            if(await sharp.resize(nw,nh).toFile(thumbnailFile))
            {
                doc.thumbnail={
                    fileName:path.basename(thumbnailFile),
                    filePath:fullPath.substr(getFullPath('').length+1),
                    fileSize:0,
                    fileUrl:"",
                    attributes:{
                        width:nw,
                        height:nh
                    }
                };
                
                doc.thumbnail.fileUrl=doc.thumbnail.filePath.substr(0,7)=='public/'?doc.thumbnail.filePath.substr(7):doc.thumbnail.filePath;
            }
        }
    }

    return await (await asyncAttachments()).create(doc);
  }
}

src/pages/api/attachments.js

import nc from 'next-connect';
import {join as pathJoin,dirname} from 'path';
import { openSync, closeSync, appendFileSync,existsSync as fsExists,renameSync,chmodSync,rmdirSync } from 'fs';
import {getFullPath} from 'lib/api-helpers';
import {mkdir} from 'shelljs';
import {platform} from 'os';
import asyncAtt, {createFromFile} from 'models/Attachments';
import { ObjectId } from 'mongodb';

const handler=nc();
handler.post(async function(req,res){
  if(req.session.files==undefined) req.session.files={};

  let fd=0,sess;
  let sessFiles=req.session.get('files');
  const autoAttach=typeof req.body['autoAttach']=='boolean'?req.body.autoAttach:true;

  if(sessFiles?req.session.files[req.body.tempId]==undefined:true)
  {
      sess={...req.body};

      if(sess['autoAttach']!=undefined) delete sess.autoAttach;

      delete sess.tempId;
      delete sess.content;
      sess.mime=sess.mime.split(';')[0];
      sess.tempFile=getFullPath(pathJoin('.tmp','uploads',req.body.tempId,sess.fileName));

      if(!sessFiles) sessFiles={};

      sessFiles[req.body.tempId]=sess;
  }else{
      sessFiles[req.body.tempId].index=req.body.index;
      sess=sessFiles[req.body.tempId]; 
  }

  req.session.set('files',sessFiles);

  if(!fsExists(dirname(sess.tempFile))) mkdir('-p',dirname(sess.tempFile));

  const buffer=Buffer.from(req.body.content,'base64');
  let result={
      index:req.body.index,
      size:buffer.length
  };
        
  fd=openSync(sess.tempFile,'as');
  appendFileSync(fd,buffer);
  //ensure file is written correctly, wait for 500ms
  await (new Promise((resolve)=>{
      setTimeout(() => {
         resolve();
      }, 500);
  }));
  closeSync(fd);
        
  if(sess.index==sess.parts-1)
  {
    if(autoAttach)
    {
        const names=sess.fileName.split('.'),now=new Date();
        const ext=names.pop(),
        //baseDir is public/uploads/YYYY-MM/
        baseDir=getFullPath(pathJoin('public','uploads',now.getFullYear()+'-'+(now.getMonth()+1).toString().padStart(2,'0')));                
            
        if(!fsExists(baseDir)) {
            await mkdir('-p',baseDir);

            if(platform()!='win32') chmodSync(baseDir,0o775);
        }

        const newFile=pathJoin(baseDir,names.join('.')+'-'+req.body.tempId+'.'+ext);
        //move uploaded file from temp directory to destined uploads dir
        renameSync(sess.tempFile,newFile);
        //remove temporary directory
        rmdirSync(dirname(sess.tempFile));

        if(platform()!='win32') chmodSync(newFile,0o664);

        delete req.session.files[req.body.tempId];

        try{
           const attachment=await createFromFile(newFile,{
                 fileSize:sess.fileSize,
                 mime:sess.mime
           });

           if(attachment) result=await attachment.safe;
        }catch(err){
           res.json({
               success:false,
               result:err
           });
           return;
        }
    }else{
        result.tempFile=sess.tempFile;
    }
 }

 res.status(200).json({
    success:true,
    result
 });
});
export default handler;

Original MP4 file:

Uploaded MP4 file:

Original APK file:

Uploaded APK file:

Is something missing? Thanks in advance anyway

Solution

Finally I have found my own answer. Slice the file, convert it to BASE64, and then upload it. With this, it will send an exact copy, bytes per bytes. I'll try to make a node module for chunked file upload.