Search code examples
c++c++-clifido-u2fyubico

Return from native code to managed corrupts the returned object


I'm trying to build an implementation for U2F authentication in my WPF program using C++/CLI and yubico's libu2f-host(https://github.com/Yubico/libu2f-host) I have a fully native function in my C++/CLI dll which calls the appropriate libu2f functions and it then returns a pointer to u2f_reply(my own class) which contains the response and the error code. However when this object is received by the managed part of the dll the response code is still correct yet the response string seems to have become fully corrupted.

I tried returning the char pointer immediately and then turn it into a managed string using marshal and I've tried to use a standard C# string. I tried using memcpy to copy the response into reply->response but without luck.

Managed_U2F_Reply^ Utilities::AuthenticateU2F(System::String^ challenge)
{
    IntPtr ptrToChallenge = System::Runtime::InteropServices::Marshal::StringToHGlobalAnsi(challenge);
    u2f_reply* reply = NativeUtilities::AuthenticateU2F(static_cast<char*>(ptrToChallenge.ToPointer()));
    //clean up before return
    System::Runtime::InteropServices::Marshal::FreeHGlobal(ptrToChallenge);
    Managed_U2F_Reply^ managedReply = gcnew Managed_U2F_Reply();
    managedReply->ResultCode = reply->responseCode;
    if (managedReply->ResultCode == 0)
    {
        managedReply->Response = gcnew String(reply->response, 0, reply->length);
    }
    return managedReply;
}
struct u2f_reply
{
    int responseCode;
    int length;
    char* response;
};
u2f_reply* NativeUtilities::AuthenticateU2F(char* challenge)
{
    u2f_reply* reply = new u2f_reply();
    char response[2048] = { 0 };
    size_t response_len = sizeof(response);
    u2fh_devs* devs = NULL;
    u2fh_rc rc;
    u2fh_initflags flags = (u2fh_initflags)0;
    rc = u2fh_global_init(flags);
    if (rc != U2FH_OK)
    {
        reply->responseCode = rc;
        return reply;
    }
    rc = u2fh_devs_init(&devs);
    if (rc != U2FH_OK)
    {
        reply->responseCode = rc;
        return reply;
    }
    rc = u2fh_devs_discover(devs, NULL);
    if (rc != U2FH_OK)
    {
        reply->responseCode = rc;
        return reply;
    }
    rc = u2fh_authenticate2(devs, challenge, "https://demo.yubico.com", response, &response_len, U2FH_REQUEST_USER_PRESENCE);
    if (rc != U2FH_OK)
    {
        reply->responseCode = rc;
        return reply;
    }
    u2fh_devs_done(devs);
    u2fh_global_done();
    reply->length = response_len;
    reply->response = response;//adding printf here prints it out correctly, yet it doesn't when I call it in Utilities::AuthenticateU2F
    reply->responseCode = rc;
    return reply;
}

I expect it to return a json object with the response data, as here: https://developers.yubico.com/libu2f-host/, yet I get a random string of ASCII characters(can't paste it here) and no error


Solution

  • Thanks to Hans Passant I fixed the issue by using:

    u2f_reply* NativeUtilities::AuthenticateU2F(char* challenge)
    {
        u2f_reply* reply = new u2f_reply();
        char response[2048] = { 0 };
        size_t response_len = sizeof(response);
        u2fh_devs* devs = NULL;
        u2fh_rc rc;
        u2fh_initflags flags = (u2fh_initflags)0;
        rc = u2fh_global_init(flags);
        if (rc != U2FH_OK)
        {
            reply->responseCode = rc;
            return reply;
        }
        rc = u2fh_devs_init(&devs);
        if (rc != U2FH_OK)
        {
            reply->responseCode = rc;
            return reply;
        }
        rc = u2fh_devs_discover(devs, NULL);
        if (rc != U2FH_OK)
        {
            reply->responseCode = rc;
            return reply;
        }
        rc = u2fh_authenticate2(devs, challenge, "https://demo.yubico.com", response, &response_len, U2FH_REQUEST_USER_PRESENCE);
        if (rc != U2FH_OK)
        {
            reply->responseCode = rc;
            return reply;
        }
        u2fh_devs_done(devs);
        u2fh_global_done();
        reply->length = response_len;
        reply->response = new char[response_len];
        memcpy_s(reply->response, response_len, response, response_len);
        reply->responseCode = rc;
        return reply;
    }