Read QR Code in Unity with ARKit and ZXing

Question

I'm trying to read a QR code with the following libraries:

• ARKit
• ZXing

However it doesn't seem to go well. After several hours I still don't manage to read out a decent QR code. When debugging i apply the texture to see my result. It looks red because of the Texture Y but other than that it shows the QR code. Interpreting the texture doesn't return any data analyzed by ZXing.

This is the following code i'm using for this:

#if UNITY_IOS && !UNITY_EDITOR
    // Update is called once per frame
    // BETTER: InvokeRepeating
    void Update()
    {
        if (!done) {
            ARTextureHandles handles = arSession.GetARVideoTextureHandles();
            //ARTextureHandles handles = UnityARSessionNativeInterface.GetARSessionNativeInterface().GetARVideoTextureHandles();
            if (handles.IsNull())
            {
                return;
            }
            if (handles.TextureY != System.IntPtr.Zero) {
                ReadQRCode (handles.TextureY);
            }
        }

}
#endif

private void ReadQRCode(System.IntPtr mtlTexPtr)
{
    Debug.Log("---------------");
    Debug.Log("Scanning...");

    Resolution currentResolution = Screen.currentResolution;

    tex = (UnityEngine.Texture2D)GameObject.Find("Camera").GetComponent<UnityARVideo>().m_ClearMaterial.GetTexture("_textureCbCr");

    tex.UpdateExternalTexture(mtlTexPtr);

    try
    {
        if(barCodeReader == null) {
            Debug.Log("Could not find barcorereader");
        }
        if(tex == null) {
            Debug.Log("Could not find texture");
        }

        var data = barCodeReader.Decode(tex.GetPixels32(), currentResolution.width, currentResolution.height);
        if (data != null)
        {
            Debug.Log("QR: " + data.Text);
        }
        else
        {
            Debug.Log("NO QR: " + "No QR code detected !");
        }
    }
    catch (Exception e)
    {
        Debug.LogError("Error reading QR");
        Debug.LogError(e.Message);
    }
}

Answer 1

After some further digging i came across an example that had something similar for OpenCV.

This proven to work for me with a good speed.

public class FrameCapturer : MonoBehaviour {

    // Script Inputs
    public bool m_shouldCaptureOnNextFrame = false;
    public Color32[] m_lastCapturedColors;

    // Privates
    Texture2D m_centerPixTex;

    void Start()
    {
        Resolution currentResolution = Screen.currentResolution;
        m_centerPixTex = new Texture2D(currentResolution.width, currentResolution.height, TextureFormat.RGBA32, false);
    }

    void OnPostRender()
    {

        if (m_shouldCaptureOnNextFrame)
        {
            Resolution res = Screen.currentResolution;
            m_lastCapturedColors = GetRenderedColors();
            m_shouldCaptureOnNextFrame = false;
        }
    }

    // Helpers
    Color32[] GetRenderedColors()
    {
        Resolution currentResolution = Screen.currentResolution;
        m_centerPixTex.ReadPixels(new Rect(0, 0, currentResolution.width, currentResolution.height), 0, 0);
        m_centerPixTex.Apply();

        return m_centerPixTex.GetPixels32();
    }
}

I attached it to the main camera where the AR scripts where also attached under. Then i the qr code reader i could simply use the following:

public class QRCodeReader : MonoBehaviour
{
    public Camera cam;
    private BarcodeReader barCodeReader;

    FrameCapturer m_pixelCapturer;

    // Use this for initialization
    void Start()
    {
        barCodeReader = new BarcodeReader();
        Resolution currentResolution = Screen.currentResolution;
        m_pixelCapturer = cam.GetComponent<FrameCapturer>();
    }

    void Update()
    {

        Resolution currentResolution = Screen.currentResolution;

        try
        {
            Color32[] framebuffer = m_pixelCapturer.m_lastCapturedColors;
            if (framebuffer.Length == 0)
            {
                return;
            }

            var data = barCodeReader.Decode(framebuffer, currentResolution.width, currentResolution.height);
            if (data != null)
            {
                // QRCode detected.
                Debug.Log(data);
                Debug.Log("QR: " + data.Text);

                //OnQrCodeRead(new QrCodeReadEventArgs() { text = data.Text });
            }

        }
        catch (Exception e)
        {
            Debug.LogError("Error reading QR");
            Debug.LogError(e.Message);
        }

        // skip 1 frame each time 
        // solves GetPixels() blocks for ReadPixels() to complete
        // https://medium.com/google-developers/real-time-image-capture-in-unity-458de1364a4c
        m_pixelCapturer.m_shouldCaptureOnNextFrame = true;

    }
}

original source where i adapted this answer to: https://github.com/realityenhanced/ARKitExperiments/blob/master/Assets/Scripts/CaptureCenterPixel.cs