Hey I am working on a project(which is in swift) and it compares two audio signals and measure the correctness. AUDIOKIT pod is used to convert the audio from microphone(AKAmplitudeTracker) to float numbers. I am trying to implement the same method by applying the tracker on the AKAudioPlayer. What I am trying to do is performing sampling on the source signal and the reference signal and get it as amplitude data only, and then performing DTW(Dynamic time warping) algorithm.
Is there any means by which I can get the AKAudioPlayer music to be converted as amplitude data? Is it possible to add a tracker to the AKAudioPlayer currently playing music? Codes are given below. I need some expert advices, Thanks in advance and Happy coding.
//
// Conductor.swift
// AmplitudeTracker
//
// Created by Mark Jeschke on 10/3/17.
// Copyright © 2017 Mark Jeschke. All rights reserved.
//
import AudioKit
import AudioKitUI
// Treat the conductor like a manager for the audio engine.
class Conductor {
// Singleton of the Conductor class to avoid multiple instances of the audio engine
var url:URL?
var fileName:String?
var type:String?
static let sharedInstance = Conductor()
var isPlayingKit:Bool?
var micTracker: AKAmplitudeTracker!
var mp3Tracker: AKAmplitudeTracker!
var player:AKAudioPlayer!
var mic: AKMicrophone!
var delay: AKDelay!
var reverb: AKCostelloReverb!
// Balance between the delay and reverb mix.
var reverbAmountMixer = AKDryWetMixer()
func play(file: String, type: String) -> AKAudioPlayer? {
let url = Bundle.main.url(forResource: file, withExtension: type)
let file = try! AKAudioFile(forReading: url!)
player = try! AKAudioPlayer(file: file)
if self.isPlayingKit! {
player.play()
mp3Tracker = AKAmplitudeTracker(player)
delay = AKDelay(mp3Tracker)
delay.time = 0.0
delay.feedback = 0.0
delay.dryWetMix = 0.5
reverb = AKCostelloReverb(delay)
reverb.presetShortTailCostelloReverb()
reverbAmountMixer = AKDryWetMixer(delay, reverb, balance: 0.8)
AudioKit.output = reverbAmountMixer
}
else {
self.isPlayingKit = true
AudioKit.output = nil
player.stop()
}
return player
}
init() {
AKSettings.playbackWhileMuted = true
mic = AKMicrophone()
print("INIT CONDUCTOR")
micTracker = AKAmplitudeTracker(mic)
delay = AKDelay(micTracker)
delay.time = 0.5
delay.feedback = 0.1
delay.dryWetMix = 0.5
reverb = AKCostelloReverb(delay)
reverb.presetShortTailCostelloReverb()
reverbAmountMixer = AKDryWetMixer(delay, reverb, balance: 0.8)
AudioKit.output = reverbAmountMixer
isPlayingKit = true
startAudioEngine()
}
func startAudioEngine() {
AudioKit.start()
isPlayingKit = true
print("Audio engine started")
}
func stopAudioEngine() {
AudioKit.stop()
isPlayingKit = false
print("Audio engine stopped")
}
}
The above mentioned method captures the amplitude of the microphone.
The below given is the location where I tried to use AKAmplitudeTracker on AKAudioPlayer.
//
// ViewController.swift
// AudioBoom
//
// Created by Alex Babu on 20/06/18.
// Copyright © 2018 Naico. All rights reserved.
//
import AudioKit
class ViewController: UIViewController {
var instantanousAmplitudeData = [Double]()
var timer:Timer?
var timerCount:Int?
let conductor = Conductor.sharedInstance
var player:AKAudioPlayer?
@IBOutlet weak var holderView: UIView!
@IBOutlet weak var equalizer: UILabel!
@IBOutlet weak var percentageLabel: UILabel!
override func viewDidLoad() {
super.viewDidLoad()
timerCount = 0
playMusicOutlet.layer.cornerRadius = playMusicOutlet.frame.size.height/2
playMusicOutlet.layer.borderColor = UIColor.cyan.cgColor
playMusicOutlet.layer.borderWidth = 2.0
playMusicOutlet.clipsToBounds = true
musicDTW.layer.cornerRadius = musicDTW.frame.size.height/2
musicDTW.layer.borderColor = UIColor.cyan.cgColor
musicDTW.layer.borderWidth = 2.0
musicDTW.clipsToBounds = true
holderView.layer.cornerRadius = holderView.frame.size.width/2
holderView.clipsToBounds = true
}
override var preferredStatusBarStyle: UIStatusBarStyle {
return .lightContent
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
}
@IBOutlet weak var playMusicOutlet: UIButton!
@IBAction func playMusic(_ sender: Any) {
playMusicOutlet.setTitle("Talk now", for: .normal)
self.equalizer.isHidden = false
timerCount = 0
AVAudioSession.sharedInstance().requestRecordPermission({(_ granted: Bool) -> Void in
if granted {
print("Permission granted")
self.timer = Timer.scheduledTimer(withTimeInterval: 0.05, repeats: true) { [unowned self] (timer) in
if let count = self.timerCount {
DispatchQueue.main.async {
self.timerCount = count + 1
print("Amplitude of mic detected:\(self.conductor.micTracker.amplitude)")
print("Amplitude of mic counter:\(String(describing: count))")
print("Amplitude of mp3 detected:\(self.conductor.micTracker.amplitude)")
print("Amplitude of mp3 Counter:\(String(describing: count))")
self.instantanousAmplitudeData.append(self.conductor.micTracker.amplitude)
self.equalizer.frame.size.height = CGFloat(self.conductor.micTracker.amplitude * 500)
self.percentageLabel.text = String(Int(((self.conductor.micTracker.amplitude * 500)/500) * 100)) + "%"
if count == 10000 {
timer.invalidate()
self.equalizer.isHidden = true
}
}
}
}
}
else {
print("Permission denied")
}
})
}
@IBOutlet weak var musicDTW: UIButton!
@IBAction func musicDTWAction(_ sender: Any) {
let anotherConductor = Conductor.sharedInstance
if let ccc = anotherConductor.play(file: "Timebomb", type: "mp3") {
musicDTW.setTitle("Music DTW on", for: .normal)
if let mp3Tracker = conductor.mp3Tracker {
self.equalizer.frame.size.height = CGFloat(mp3Tracker.amplitude * 500)
}
}
else {
musicDTW.setTitle("Music DTW off", for: .normal)
}
}
}
Try these out!
Conductor Class
import AudioKit
import AudioKitUI
// Treat the conductor like a manager for the audio engine.
class Conductor {
// Singleton of the Conductor class to avoid multiple instances of the audio engine
var url:URL?
var fileName:String?
var type:String?
static let sharedInstance = Conductor()
var isPlayingKit:Bool?
var micTracker: AKAmplitudeTracker!
var mp3Tracker: AKAmplitudeTracker!
var player:AKAudioPlayer!
var mic: AKMicrophone!
var delay: AKDelay!
var reverb: AKCostelloReverb!
// Balance between the delay and reverb mix.
var reverbAmountMixer = AKDryWetMixer()
func play(file: String, type: String) -> AKAudioPlayer? {
let url = Bundle.main.url(forResource: file, withExtension: type)
let file = try! AKAudioFile(forReading: url!)
player = try! AKAudioPlayer(file: file)
if self.isPlayingKit! {
mp3Tracker = AKAmplitudeTracker(player)
delay = AKDelay(mp3Tracker)
delay.time = 0.0
delay.feedback = 0.0
delay.dryWetMix = 0.5
reverb = AKCostelloReverb(delay)
reverb.presetShortTailCostelloReverb()
reverbAmountMixer = AKDryWetMixer(delay, reverb, balance: 0.8)
AudioKit.output = reverbAmountMixer //#1
player.play() //#2
}
else {
self.isPlayingKit = true
AudioKit.output = nil
// player.stop()
stopAudioEngine()
}
return player
}
func isPlayingAudioKit() -> Bool {
return isPlayingKit!
}
init() {
self.isPlayingKit = false
}
func initMicrophone() {
AKSettings.playbackWhileMuted = true
mic = AKMicrophone()
print("INIT CONDUCTOR")
micTracker = AKAmplitudeTracker(mic)
delay = AKDelay(micTracker)
delay.time = 1.5
delay.feedback = 0.1
delay.dryWetMix = 1.0
reverb = AKCostelloReverb(delay)
reverb.presetShortTailCostelloReverb()
reverbAmountMixer = AKDryWetMixer(delay, reverb, balance: 0.5)
AudioKit.output = reverbAmountMixer
isPlayingKit = true
startAudioEngine()
}
func startAudioEngine() {
AudioKit.start()
isPlayingKit = true
print("Audio engine started")
}
func stopAudioEngine() {
AudioKit.stop()
isPlayingKit = false
print("Audio engine stopped")
}
}
ViewController
import AudioKit
class ViewController: UIViewController {
var instantanousUserAudioData = [Float]()
var referenceAudioData = [Float]()
var timer:Timer?
var timerCount:Int?
let conductor = Conductor.sharedInstance
@IBOutlet weak var holderView: UIView!
@IBOutlet weak var equalizer: UILabel!
@IBOutlet weak var percentageLabel: UILabel!
@IBOutlet weak var timerOutlet: UIButton!
@IBOutlet weak var micOutlet: UIButton!
@IBOutlet weak var DTWOutlet: UIButton!
@IBOutlet weak var musicOutlet: UIButton!
@IBOutlet weak var resultLabel: UILabel!
@IBAction func timerAction(_ sender: Any) {
self.timer?.invalidate()
}
override func viewDidLoad() {
super.viewDidLoad()
timerCount = 0
micOutlet.layer.cornerRadius = micOutlet.frame.size.height/2
micOutlet.layer.borderColor = UIColor.cyan.cgColor
micOutlet.layer.borderWidth = 2.0
micOutlet.clipsToBounds = true
musicOutlet.layer.cornerRadius = musicOutlet.frame.size.height/2
musicOutlet.layer.borderColor = UIColor.cyan.cgColor
musicOutlet.layer.borderWidth = 2.0
musicOutlet.clipsToBounds = true
DTWOutlet.layer.cornerRadius = DTWOutlet.frame.size.height/2
DTWOutlet.layer.borderColor = UIColor.cyan.cgColor
DTWOutlet.layer.borderWidth = 2.0
DTWOutlet.clipsToBounds = true
timerOutlet.layer.cornerRadius = timerOutlet.frame.size.height/2
timerOutlet.layer.borderColor = UIColor.cyan.cgColor
timerOutlet.layer.borderWidth = 2.0
timerOutlet.clipsToBounds = true
holderView.layer.cornerRadius = holderView.frame.size.width/2
holderView.clipsToBounds = true
self.micOutlet.isEnabled = false
self.musicOutlet.isEnabled = false
AVAudioSession.sharedInstance().requestRecordPermission({(_ granted: Bool) -> Void in
self.micOutlet.isEnabled = granted
self.musicOutlet.isEnabled = granted
})
}
override var preferredStatusBarStyle: UIStatusBarStyle {
return .lightContent
}
override func didReceiveMemoryWarning() {
super.didReceiveMemoryWarning()
}
@IBAction func micAction(_ sender: Any) {
conductor.initMicrophone()
self.timerCount = 0
self.equalizer.isHidden = false
self.timer = Timer.scheduledTimer(withTimeInterval: 0.05, repeats: true) { [unowned self] (timer) in
if let count = self.timerCount {
DispatchQueue.main.async {
self.timerCount = count + 1
print("Amplitude of mic detected:\(self.conductor.micTracker.amplitude)")
print("Amplitude of mp3 Counter:\(String(describing: count))")
self.instantanousUserAudioData.append(Float(self.conductor.micTracker.amplitude))
self.equalizer.frame.size.height = CGFloat(self.conductor.micTracker.amplitude * 500)
self.percentageLabel.text = String(Int(((self.conductor.micTracker.amplitude * 500)/500) * 100)) + "%"
if count > 10 && self.conductor.micTracker.amplitude == 0.0 && self.instantanousUserAudioData.last == 0.0 {
self.micOutlet.backgroundColor = .green
self.micOutlet.setTitleColor(.black, for: .normal)
self.micOutlet.layer.borderColor = UIColor.red.cgColor
timer.invalidate()
}
if count == 0 {
self.micOutlet.backgroundColor = .clear
self.micOutlet.setTitleColor(.cyan, for: .normal)
self.micOutlet.layer.borderColor = UIColor.cyan.cgColor
}
}
}
}
}
@IBAction func musicAction(_ sender: Any) {
self.timerCount = 0
if self.conductor.play(file: voiceReference, type: type_mp3) != nil {
self.timer?.invalidate()
self.timer = Timer.scheduledTimer(withTimeInterval: 0.05, repeats: true) { [unowned self] (timer) in
if let count = self.timerCount {
DispatchQueue.main.async {
self.timerCount = count + 1
print("Amplitude of mp3 detected:\(self.conductor.mp3Tracker.amplitude)")
print("Amplitude of mp3 Counter:\(String(describing: count))")
self.referenceAudioData.append(Float(self.conductor.mp3Tracker.amplitude))
self.equalizer.frame.size.height = CGFloat(self.conductor.mp3Tracker.amplitude * 500)
self.equalizer.isHidden = false
self.percentageLabel.text = String(Int(((self.conductor.mp3Tracker.amplitude * 500)/500) * 100)) + "%"
if count > 10 && self.conductor.mp3Tracker.amplitude == 0.0 && self.referenceAudioData.last == 0.0 {
self.musicOutlet.backgroundColor = .green
self.musicOutlet.setTitleColor(.black, for: .normal)
self.musicOutlet.layer.borderColor = UIColor.red.cgColor
timer.invalidate()
}
if count == 0 {
self.musicOutlet.backgroundColor = .clear
self.musicOutlet.setTitleColor(.cyan, for: .normal)
self.musicOutlet.layer.borderColor = UIColor.cyan.cgColor
}
}
}
}
}
else {
}
}
@IBAction func resultAction(_ sender: Any) {
print("mic array:\(instantanousUserAudioData)")
print("song array:\(referenceAudioData)")
self.timer?.invalidate()
if referenceAudioData.count > 0, instantanousUserAudioData.count > 0 {
let refData = knn_curve_label_pair(curve: referenceAudioData,label: "reference")
let userData = knn_curve_label_pair(curve: instantanousUserAudioData,label: "userData")
let attempt:KNNDTW = KNNDTW()
attempt.train(data_sets: [refData,userData])
let prediction: knn_certainty_label_pair = attempt.predict(curve_to_test: referenceAudioData)
print("predicted :" + prediction.label, "with ", prediction.probability * 100,"% certainty")
resultLabel.text = "DTW cost is " + String(attempt.dtw_cost(y: referenceAudioData, x: instantanousUserAudioData))
print("COST OF THE DTW ALGORITHM IS : \(String(attempt.dtw_cost(y: referenceAudioData, x: instantanousUserAudioData)))")
}
}
}