I have an electron program that selects multiple regions of a landscape video and lets you rearrange them in a portrait canvas. I'm having trouble building the proper ffmpeg command to create the video. I have this somewhat working. I can export 2 layers, but i can't export if i only have 1 layer or if i have 3 or more layers selected.
2 regions of video selected
layers [
{ top: 0, left: 658, width: 576, height: 1080 },
{ top: 262, left: 0, width: 576, height: 324 }
]
newPositions [
{ top: 0, left: 0, width: 576, height: 1080 },
{ top: 0, left: 0, width: 576, height: 324 }
]
filtergraph [0]crop=576:1080:658:0,scale=576:1080[v0];[0]crop=576:324:0:262,scale=576:324[v1];[v0][v1]overlay=0:0:0:0[out]
No Error export successful
1 region selected
layers [ { top: 0, left: 650, width: 576, height: 1080 } ]
newPositions [ { top: 0, left: 0, width: 576, height: 1080 } ]
filtergraph [0]crop=576:1080:650:0,scale=576:1080[v0];[v0]overlay=0:0[out]
FFmpeg error: [fc#0 @ 000001dd3b6db0c0] Cannot find a matching stream for unlabeled input pad overlay
Error initializing complex filters: Invalid argument
3 regions of video selected
layers [
{ top: 0, left: 641, width: 576, height: 1080 },
{ top: 250, left: 0, width: 576, height: 324 },
{ top: 756, left: 0, width: 576, height: 324 }
]
newPositions [
{ top: 0, left: 0, width: 576, height: 1080 },
{ top: 0, left: 0, width: 576, height: 324 },
{ top: 756, left: 0, width: 576, height: 324 }
]
filtergraph [0]crop=576:1080:641:0,scale=576:1080[v0];[0]crop=576:324:0:250,scale=576:324[v1];[0]crop=576:324:0:756,scale=576:324[v2];[v0][v1][v2]overlay=0:0:0:0:0:756[out]
FFmpeg error: [AVFilterGraph @ 0000018faf2189c0] More input link labels specified for filter 'overlay' than it has inputs: 3 > 2
[AVFilterGraph @ 0000018faf2189c0] Error linking filters
FFmpeg error: Failed to set value '[0]crop=576:1080:698:0,scale=576:1080[v0];[0]crop=576:324:0:264,scale=576:324[v1];[0]crop=576:324:0:756,scale=576:324[v2];[v0][v1][v2]overlay=0:0:0:0:0:0[out]' for option 'filter_complex': Invalid argument
Error parsing global options: Invalid argument
I can't figure out how to construct the proper overlay command. Here is the js code i'm using from my electron app.
ipcMain.handle('export-video', async (_event, args) => {
const { videoFile, outputName, layers, newPositions } = args;
const ffmpegPath = path.join(__dirname, 'bin', 'ffmpeg');
const outputDir = checkOutputDir();
// use same video for each layer as input
// crop, scale, and position each layer
// overlay each layer on top of each other
// export video
console.log('layers', layers);
console.log('newPositions', newPositions);
let filtergraph = '';
for (let i = 0; i < layers.length; i++) {
const { top, left, width, height } = layers[i];
const { width: newWidth, height: newHeight } = newPositions[i];
const filter = `[0]crop=${width}:${height}:${left}:${top},scale=${newWidth}:${newHeight}[v${i}];`;
filtergraph += filter;
}
for (let i = 0; i < layers.length; i++) {
const filter = `[v${i}]`;
filtergraph += filter;
}
filtergraph += `overlay=`;
for (let i = 0; i < layers.length; i++) {
const { top: newTop, left: newLeft } = newPositions[i];
const overlay = `${newLeft}:${newTop}:`;
filtergraph += overlay;
}
filtergraph = filtergraph.slice(0, -1); // remove last comma
filtergraph += `[out]`;
console.log('filtergraph', filtergraph);
const ffmpeg = spawn(ffmpegPath, [
'-i', videoFile,
'-filter_complex', filtergraph,
'-map', '[out]',
'-c:v', 'libx264',
'-preset', 'ultrafast',
'-crf', '18',
'-y',
path.join(outputDir, `${outputName}`)
]);
ffmpeg.stdout.on('data', (data) => {
console.log(`FFmpeg output: ${data}`);
});
ffmpeg.stderr.on('data', (data) => {
console.error(`FFmpeg error: ${data}`);
});
ffmpeg.on('close', (code) => {
console.log(`FFmpeg process exited with code ${code}`);
// event.reply('ffmpeg-export-done'); // Notify the renderer process
});
});
Any advice might be helpful, The docs are confusing, Thanks.
Edit I'm getting closer with this Output:
layers [
{ top: 0, left: 677, width: 576, height: 1080 },
{ top: 240, left: 0, width: 576, height: 324 }
]
newPositions [
{ top: 0, left: 0, width: 576, height: 1080 },
{ top: 0, left: 0, width: 576, height: 324 }
]
filtergraph [0]crop=576:1080:677:0,scale=576:1080[v0];[0]crop=576:324:0:240,scale=576:324[v1];[0][v0]overlay=0:0[o0];[o0][v1]overlay=0:0[o1]
ipcMain.handle('export-video', async (_event, args) => {
const { videoFile, outputName, layers, newPositions } = args;
const ffmpegPath = path.join(__dirname, 'bin', 'ffmpeg');
const outputDir = checkOutputDir();
// use same video for each layer as input
// crop, scale, and position each layer
// overlay each layer on top of each other
// export video
console.log('layers', layers);
console.log('newPositions', newPositions);
let filtergraph = '';
for (let i = 0; i < layers.length; i++) {
const { top, left, width, height } = layers[i];
const { width: newWidth, height: newHeight } = newPositions[i];
const filter = `[0]crop=${width}:${height}:${left}:${top},scale=${newWidth}:${newHeight}[v${i}];`;
filtergraph += filter;
}
for (let i = 0; i < layers.length; i++) {
if (i === 0) {
filtergraph += `[0][v${i}]overlay=`;
} else {
filtergraph += `[o${i-1}][v${i}]overlay=`;
}
const { top: newTop, left: newLeft } = newPositions[i];
let overlay = '';
if (i !== layers.length - 1) {
overlay = `${newLeft}:${newTop}[o${i}];`;
} else {
overlay = `${newLeft}:${newTop};`;
}
filtergraph += overlay;
}
filtergraph = filtergraph.slice(0, -1); // remove last comma
filtergraph += `[o${layers.length-1}]`;
console.log('filtergraph', filtergraph);
const ffmpeg = spawn(ffmpegPath, [
'-i', videoFile,
'-filter_complex', filtergraph,
'-map', `[o${layers.length-1}]`,
'-c:v', 'libx264',
'-preset', 'ultrafast',
'-crf', '18',
'-y',
path.join(outputDir, `${outputName}`)
]);
ffmpeg.stdout.on('data', (data) => {
console.log(`FFmpeg output: ${data}`);
});
ffmpeg.stderr.on('data', (data) => {
console.error(`FFmpeg error: ${data}`);
});
ffmpeg.on('close', (code) => {
console.log(`FFmpeg process exited with code ${code}`);
// event.reply('ffmpeg-export-done'); // Notify the renderer process
});
});
The problem I'm having now is that its overlaying the regions over the original input and keeping the landscape dimensions instead of making a portrait video.
Figured it out.
1 region
layers [ { top: 0, left: 672, width: 576, height: 1080 } ]
newPositions [ { top: 0, left: 0, width: 576, height: 1080 } ]
filtergraph [0]crop=576:1080:672:0,scale=576:1080[l0]
2 regions
layers [
{ top: 0, left: 660, width: 576, height: 1080 },
{ top: 242, left: 0, width: 576, height: 324 }
]
newPositions [
{ top: 0, left: 0, width: 576, height: 1080 },
{ top: 0, left: 0, width: 576, height: 324 }
]
filtergraph [0]crop=576:1080:660:0,scale=576:1080[l0];[0]crop=576:324:0:242,scale=576:324[l1];[l0][l1]overlay=0:0[o0]
3 or more regions
layers [
{ top: 0, left: 665, width: 576, height: 1080 },
{ top: 238, left: 0, width: 576, height: 324 },
{ top: 756, left: 0, width: 576, height: 324 }
]
newPositions [
{ top: 0, left: 0, width: 576, height: 1080 },
{ top: 0, left: 0, width: 576, height: 324 },
{ top: 756, left: 0, width: 576, height: 324 }
]
filtergraph [0]crop=576:1080:665:0,scale=576:1080[l0];[0]crop=576:324:0:238,scale=576:324[l1];[0]crop=576:324:0:756,scale=576:324[l2];[l0][l1]overlay=0:0[o0];[o0][l2]overlay=0:756[o1]
Export Handler
ipcMain.handle('export-video', async (_event, args) => {
const { videoFile, outputName, layers, newPositions } = args;
const ffmpegPath = path.join(__dirname, 'bin', 'ffmpeg');
const outputDir = checkOutputDir();
console.log('layers', layers);
console.log('newPositions', newPositions);
let filtergraph = '';
let overlayCount = -1;
for (let i = 0; i < layers.length; i++) {
const { top, left, width, height } = layers[i];
const { width: newWidth, height: newHeight } = newPositions[i];
let filter = '';
filter = `[0]crop=${width}:${height}:${left}:${top},scale=${newWidth}:${newHeight}[l${i}];`;
filtergraph += filter;
}
let newLeft = 0;
let newTop = 0;
for (let i = 0; i < layers.length; i++) {
if (i === 0 && layers.length > 1) {
newLeft = newPositions[i+1].left;
newTop = newPositions[i+1].top;
filtergraph += `[l${i}][l${i+1}]overlay=${newLeft}:${newTop}[o${i}];`;
overlayCount++;
} else if (i < layers.length - 1) {
newLeft = newPositions[i+1].left;
newTop = newPositions[i+1].top;
filtergraph += `[o${i-1}][l${i+1}]overlay=${newLeft}:${newTop}[o${i}];`;
overlayCount++;
}
}
filtergraph = filtergraph.slice(0, -1); // remove last semicolon
console.log('filtergraph', filtergraph);
const ffmpeg = spawn(ffmpegPath, [
'-i', videoFile,
'-filter_complex', filtergraph,
'-map', overlayCount > -1 ? `[o${overlayCount}]` : `[l0]`,
'-c:v', 'libx264',
'-preset', 'ultrafast',
'-crf', '18',
'-y', path.join(outputDir, outputName)
]);
ffmpeg.stdout.on('data', (data) => {
console.log(`FFmpeg output: ${data}`);
});
ffmpeg.stderr.on('data', (data) => {
console.error(`FFmpeg error: ${data}`);
});
ffmpeg.on('close', (code) => {
console.log(`FFmpeg process exited with code ${code}`);
});
});