I'm trying to build a swift function to apply a color matrix filter to an image using vImageMatrixMultiply_ARGB8888 of Accellerate framework. I was previously able to apply color matrix to an image using CIFilter CIColorMatrix.
let ciimage = CIImage(image: image)!
let colorMatrix = CIFilter(name: "CIColorMatrix")!
colorMatrix.setDefaults()
colorMatrix.setValue(ciimage, forKey: "inputImage")
colorMatrix.setValue(CIVector(x: 1.9692307692307693, y: 0, z: 0, w: 0), forKey: "inputRVector")
colorMatrix.setValue(CIVector(x: 0, y: 2.226086956521739, z: 0, w: 0), forKey: "inputGVector")
colorMatrix.setValue(CIVector(x: 0, y: 0, z: 2.585858585858586, w: 0), forKey: "inputBVector")
colorMatrix.setValue(CIVector(x: 0, y: 0, z: 0, w: 1), forKey: "inputAVector")
return UIImage(ciImage: colorMatrix.outputImage!)
RotatioMatrix used in vImageMatrixMultiply_ARGB8888 use same values of input vectors of CIFilter.
rotationMatrix = [1.9692307692307693, 0, 0, 0
0, 2.226086956521739, 0, 0,
0, 0, 2.585858585858586, 0,
0, 0, 0, 1].map {
return Int16(Float($0) * Float(divisor)) // divisor = 256
}
The following code does not produce the same result as in CIFilter above.
private func rgbAdjustmentWithMatrix(image sourceImage: UIImage, rotationMatrix: [Int16]) -> UIImage? {
guard
let cgImage = sourceImage.cgImage,
let sourceImageFormat = vImage_CGImageFormat(cgImage: cgImage),
let rgbDestinationImageFormat = vImage_CGImageFormat(
bitsPerComponent: 8,
bitsPerPixel: 32,
colorSpace: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.first.rawValue),
renderingIntent: .defaultIntent) else {
print("Unable to initialize cgImage or colorSpace.")
return nil
}
guard
let sourceBuffer = try? vImage_Buffer(cgImage: cgImage),
var rgbDestinationBuffer = try? vImage_Buffer(width: Int(sourceBuffer.width),
height: Int(sourceBuffer.height),
bitsPerPixel: rgbDestinationImageFormat.bitsPerPixel) else {
print("Unable to initialize source buffer or destination buffer.")
return nil
}
defer {
sourceBuffer.free()
rgbDestinationBuffer.free()
}
do {
let toRgbConverter = try vImageConverter.make(sourceFormat: sourceImageFormat, destinationFormat: rgbDestinationImageFormat)
try toRgbConverter.convert(source: sourceBuffer, destination: &rgbDestinationBuffer)
} catch {
print("Unable to initialize converter or unable to convert.")
return nil
}
guard var resultBuffer = try? vImage_Buffer(width: Int(sourceBuffer.width),
height: Int(sourceBuffer.height),
bitsPerPixel: rgbDestinationImageFormat.bitsPerPixel) else {
print("Unable to initialize result buffer.")
return nil
}
defer {
resultBuffer.free()
}
var error: vImage_Error
let divisor: Int32 = 256
let preBias = [Int16](repeating: -256, count: 4)
let postBias = [Int32](repeating: 256 * divisor, count: 4)
error = vImageMatrixMultiply_ARGB8888(&rgbDestinationBuffer,
&resultBuffer,
rotationMatrix,
divisor,
preBias,
postBias,
0)
guard error == kvImageNoError else { return nil }
if let cgImage = try? resultBuffer.createCGImage(format: rgbDestinationImageFormat) {
return UIImage(cgImage: cgImage)
} else {
return nil
}
}
Could be a color space problem? Or vImageMatrixMultiply_ARGB8888 input matrix just differ from CIFilter input matrix?
So, it looks like vImageMatrixMultiply_ARGB8888 expects values in this order:
let rotationMatrix = [
1, 0, 0, 0, // A
0, 1.9692307692307693, 0, 0, // R
0, 0, 2.226086956521739, 0, // G
0, 0, 0, 2.585858585858586 // B
].map { return Int16($0 * Float(divisor)) } // divisor = 0x1000
This is inconsistent with the Apple sample code which implies BGRA. Possibly that matrix was originally meant to be used with a different pixelBuffer format - or some other issue.
I believe the other problem you're running into is that the CIFilter is using a default CIContext. When I run this I get a linearSRGB workingColorSpace.
let context = CIContext()
print(context.workingColorSpace ?? "")
Output:
<CGColorSpace 0x7fe4abd06d30> (kCGColorSpaceICCBased; kCGColorSpaceModelRGB; sRGB Linear)
So you can either specify the CGColorSpaceCreateDeviceRGB() as the CIContext workingColorSpace for your CIFilter, or set the rgbDestinationImageFormat colorSpace to .linearSRGB in your vImage usage. Between this and the corrected matrix order, I believe your output images will now match.
// error handling removed for brevity
let ciImage = CIImage(cgImage: cgImage)
let colorMatrix = CIFilter(name: "CIColorMatrix")!
colorMatrix.setDefaults()
colorMatrix.setValue(ciImage, forKey: "inputImage")
colorMatrix.setValue(CIVector(x: 1.9692307692307693, y: 0, z: 0, w: 0), forKey: "inputRVector")
colorMatrix.setValue(CIVector(x: 0, y: 2.226086956521739, z: 0, w: 0), forKey: "inputGVector")
colorMatrix.setValue(CIVector(x: 0, y: 0, z: 2.585858585858586, w: 0), forKey: "inputBVector")
colorMatrix.setValue(CIVector(x: 0, y: 0, z: 0, w: 1), forKey: "inputAVector")
let context = CIContext(options: [.workingColorSpace : CGColorSpaceCreateDeviceRGB()]) // without this, colorSpace defaults to linearSRGB
return context.createCGImage(colorMatrix.outputImage!, from: rect)!
Under this scenario, there's probably no need for pre and post bias unless you have some other reason for doing this.
Lastly, you might benefit from a larger divisor. While the pixel data is 8-bits per channel, the accumulator is 32-bit and matrix values are 16-bit. Most of the reference code uses a divisor of 0x1000 (4096).
func rgbAdjustmentWithMatrix(image sourceImage: NSImage, rotationMatrix: [Int16]) -> CGImage? {
var rect = CGRect(x: 0, y: 0, width: sourceImage.size.width, height: sourceImage.size.height)
guard
let cgImage = sourceImage.cgImage(forProposedRect: &rect, context: nil, hints: nil),
let sourceImageFormat = vImage_CGImageFormat(cgImage: cgImage),
let rgbDestinationImageFormat = vImage_CGImageFormat(
bitsPerComponent: 8,
bitsPerPixel: 32,
colorSpace: CGColorSpaceCreateDeviceRGB(),
// can match CIFilter default with this:
// colorSpace: CGColorSpace(name: CGColorSpace.linearSRGB)!,
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.first.rawValue),
renderingIntent: .defaultIntent)
else {
print("Unable to initialize cgImage or colorSpace.")
return nil
}
guard
var sourceBuffer = try? vImage_Buffer(cgImage: cgImage),
var rgbDestinationBuffer = try? vImage_Buffer(width: Int(sourceBuffer.width),
height: Int(sourceBuffer.height),
bitsPerPixel: rgbDestinationImageFormat.bitsPerPixel)
else {
fatalError("Error initializing source and destination buffers.")
}
defer {
sourceBuffer.free()
rgbDestinationBuffer.free()
}
do {
let toRgbConverter = try vImageConverter.make(sourceFormat: sourceImageFormat, destinationFormat: rgbDestinationImageFormat)
try toRgbConverter.convert(source: sourceBuffer, destination: &rgbDestinationBuffer)
} catch {
fatalError(error.localizedDescription)
}
let divisor: Int32 = 0x1000 // matrix values are 16 bit and accumulator is 32 bit
// 4096 gives us decent overhead for the matrix operation
// 12-bit color
let preBias: [Int16] = [0, 0, 0, 0] // or simply pass nil
let postBias: [Int32] = [0, 0, 0, 0] // or simply pass nil
let error = vImageMatrixMultiply_ARGB8888(&rgbDestinationBuffer,
&rgbDestinationBuffer,
rotationMatrix,
divisor,
preBias,
postBias,
vImage_Flags(kvImageNoFlags))
if error != kvImageNoError { print("Error: \(error)") }
let result = try? rgbDestinationBuffer.createCGImage(format: rgbDestinationImageFormat)
return result
}
Looking at your code and the output images, I believe you have the red and blue coefficients reversed. If so, you'd fill the vImage matrix like this:
rotationMatrix = [2.585858585858586, 0, 0, 0 // b
0, 2.226086956521739, 0, 0, // g
0, 0, 1.9692307692307693, 0, // r
0, 0, 0, 1].map { // a
return Int16(Float($0) * Float(divisor)) // divisor = 256
}
This would be consistent with Apple's vImage sample code for desaturating a selected portion of an image.
let desaturationMatrix = [
0.0722, 0.0722, 0.0722, 0,
0.7152, 0.7152, 0.7152, 0,
0.2126, 0.2126, 0.2126, 0,
0, 0, 0, 1
].map {
return Int16($0 * Float(divisor))
}
Note that the conversion coefficients for Rec709 to Luma are given as:
Y = R * 0.2126 + G * 0.7152 + B * 0.0722