Search code examples
c#unity-game-engineaudio-streaming

How can I stream audio directly from the response of an API request?


I am using an API where, when I make a POST request and send an audio file through form data, it returns a streamable audio. I don't know any method to play the audio immediately without having to wait until the entire audio file is fully downloaded.

  • The API returns an MP3 audio stream. I am able to play the audio after it's fully downloaded by using BestHTTP to call as follows (in OnRequestFinished)
private IEnumerator WaitAndSendRequest(byte[] audioData, string conversationId)
        { 
            request = new HTTPRequest(new Uri(serverUrl), HTTPMethods.Post, OnRequestFinished);
            request.SetHeader("Content-Type", "multipart/form-data");
            request.AddField("conversation_id", conversationId);
            request.AddBinaryData("audio_file", audioData, "audio_sample.mp3", "audio/mpeg");
            request.StreamChunksImmediately = true; 
            request.StreamFragmentSize = 1024;
            request.OnDownloadProgress = OnDownloadProgress;
            request.OnUploadProgress = OnUploadProgressDelegate;
            request.OnStreamingData = OnStreamingDataReceived;
            Debug.Log("Uploading audio...");
            request.Send();  
            yield return new WaitUntil(() => !isRequestInProgress); 
        } 
 private void OnDownloadProgress(HTTPRequest request, long downloaded, long downloadLength)
        { 
            float progress = (float)downloaded / downloadLength * 100f;
            //Debug.Log($"Download progress: {progress:F2}% ({downloaded} / {downloadLength} bytes)");
    
        }
        private void OnUploadProgressDelegate(HTTPRequest request, long downloaded, long downloadLength)
        { 
            float progress = (float)downloaded / downloadLength * 100f;
           // Debug.Log($"Upload progress: {progress:F2}% ({downloaded} / {downloadLength} bytes)");
    
        } 
        private bool OnStreamingDataReceived(HTTPRequest request, HTTPResponse response, byte[] dataFragment, int dataFragmentLength)
        {
            Debug.Log("dataFragmentLength: " + dataFragmentLength);
    
            StartCoroutine(PlayReceivedAudioChunk(dataFragment)); 
            return true;
        }
    
        private IEnumerator PlayReceivedAudioChunk(byte[] audioData)
        { 
            AudioClip audioClip = WavUtility.ToAudioClip(audioData, "AudioStream");
     
            if (audioClip != null)
            {
                audioSource.clip = audioClip;
                audioSource.Play();
                Debug.Log("Playing audio chunk." + audioData); 
                yield return new WaitUntil(() => !audioSource.isPlaying);
            }
            else
            {
                Debug.LogError("Failed to create audio clip from chunk.");
            }
        }
    
        private void OnRequestFinished(HTTPRequest req, HTTPResponse resp)
        {
            try
            {
                if (resp == null || !resp.IsSuccess)
                { 
                    Debug.LogError("Error uploading audio: " + (resp != null ? resp.Message : "Unknown error"));
    
                    UIManager.Instance.connectionTxt.text = "An error occurred, please help me check the network connection!.";  
    
                    return;
                }
    
                Debug.Log("Audio uploaded successfully. Processing response...");  
                byte[] responseAudioData = resp.Data;
     
               StartCoroutine(PlayReceivedAudio(responseAudioData));
            }
            finally
            { 
                isRequestInProgress = false;
            }  
        }
    
    
        private IEnumerator PlayReceivedAudio(byte[] audioData)
        { 
            string tempFilePath = Path.Combine(Application.temporaryCachePath, "processed_audio.mp3");
            File.WriteAllBytes(tempFilePath, audioData);
     
            using (UnityWebRequest www = UnityWebRequestMultimedia.GetAudioClip("file://" + tempFilePath, AudioType.MPEG))
            {
                yield return www.SendWebRequest();
    
                if (www.result == UnityWebRequest.Result.ConnectionError || www.result == UnityWebRequest.Result.ProtocolError)
                {
                    Debug.LogError("Error loading audio: " + www.error);
                    yield break;
                }
    
                AudioClip audioClip = DownloadHandlerAudioClip.GetContent(www);
     
                audioSource.clip = audioClip;
                audioSource.Play();
                Debug.Log("Playing received audio.");
                StartCoroutine(CheckAudioFinished());
            }
        }
  • However, when calling PlayReceivedAudioChunk, the audio cannot be heard. this is ToAudioClip function:
public static AudioClip ToAudioClip(byte[] data, string name)
    { 
        int sampleCount = data.Length / 2;  
        float[] samples = new float[sampleCount];
 
        for (int i = 0; i < sampleCount; i++)
        {
            samples[i] = (short)(data[i * 2] | (data[i * 2 + 1] << 8)) / 32768.0f;  
        } 
        AudioClip audioClip = AudioClip.Create(name, sampleCount, 1, 44100, false);
        audioClip.SetData(samples, 0);
        return audioClip;
    }```
Please suggest any solution that can work in Unity, many thanks.

Solution

  • Try this:

    using (HttpClient client = new HttpClient())
        {
            HttpResponseMessage response = await client.GetAsync("https://api.example.com/audio", HttpCompletionOption.ResponseHeadersRead);
            response.EnsureSuccessStatusCode();
    
            using (Stream stream = await response.Content.ReadAsStreamAsync())
            {
                var tempFilePath = Path.Combine(Path.GetTempPath(), "tempAudio.mp3");
                using (var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None))
                {
                    await stream.CopyToAsync(fileStream);
                }
    
                mediaElement.Source = new Uri(tempFilePath);
                mediaElement.Play();
            }
        }