JNA: __stdcall ESP corruption

Question

While trying to get notification from C++ app through JNA callback, I constantly see JVM crashes. Native.setProtected doesn't help. The original callback declaration is

typedef void (__stdcall *TRANS2QUIK_ORDER_STATUS_CALLBACK) (long nMode, DWORD dwTransID, double dNumber, LPCSTR ClassCode, LPCSTR SecCode, double dPrice, long nBalance, double dValue, long nIsSell, long nStatus, long nOrderDescriptor);

^^^ this is that doesn't work

I tried both StdCallLibrary.StdCallCallback which should work and simple Callback, they both failed. But after I wrote C++ wrapper, where I just call JNA callback as cdecl from inside of original C++ stdcall callback, everything started to work smoothly.

typedef void (*WRAPPED_ORDER_STATUS_CALLBACK) (double dNumber, double dPrice, long nBalance, double dValue, long nIsSell, long nStatus, long nOrderDescriptor);  

QUIKWRAP_API TRANS2QUIK_ORDER_STATUS_CALLBACK __stdcall wrapCallback(WRAPPED_ORDER_STATUS_CALLBACK);

^^^^^^ this is that works smoothly w/o problems

The question could this be a JNA bug specifically for this function signature? Because I have another stdcall callback in my code with different signature and it works just fine.

I created an isolated testcase for this issue, see below: java

import java.util.HashMap;

import com.sun.jna.Library;
import com.sun.jna.Native;
import com.sun.jna.NativeLong;
import com.sun.jna.Pointer;
import com.sun.jna.PointerType;
import com.sun.jna.platform.win32.WinDef;
import com.sun.jna.platform.win32.WinDef.DWORD;
import com.sun.jna.win32.StdCallLibrary;


public class JNATest {
    interface TestCallback extends StdCallLibrary.StdCallCallback {
        void testCallback(NativeLong nMode, WinDef.DWORD dwTransID, double dNumber, LPCSTR ClassCode, LPCSTR SecCode, double dPrice, NativeLong nBalance, double dValue, NativeLong nIsSell, NativeLong nStatus, NativeLong nOrderDescriptor);
    }

    interface JNADLL extends StdCallLibrary{
        void testCallback(TestCallback cb);
    }
    public static class LPCSTR extends PointerType {
        public LPCSTR(Pointer address) {
            super(address);
        }
        public LPCSTR() {
            super();
        }
        @Override
        public String toString() {
            return getPointer().getString(0);
        }
    };



    /**
     * @param args
     */
    public static void main(String[] args) {
        System.setProperty("jna.encoding", "Cp1251");

        HashMap<String, Object> nameMapping = new HashMap<String, Object>();

        nameMapping.put(Library.OPTION_FUNCTION_MAPPER, StdCallLibrary.FUNCTION_MAPPER);
        nameMapping.put(Library.OPTION_CALLING_CONVENTION, StdCallLibrary.STDCALL_CONVENTION);

        JNADLL JNADll = (JNADLL) Native.loadLibrary("QuikWrap", JNADLL.class, nameMapping);

        TestCallback cb = new TestCallback() {

            @Override
            public void testCallback(NativeLong nMode, DWORD dwTransID,
                    double dNumber, JNATest.LPCSTR ClassCode, JNATest.LPCSTR SecCode,
                    double dPrice, NativeLong nBalance, double dValue,
                    NativeLong nIsSell, NativeLong nStatus, NativeLong nOrderDescriptor) {
                System.out.println("testCallback  \n" +
                        "  nMode: " + nMode + "\n" +
                        "  dwTransID: " + dwTransID + "\n" +
                        "  number: " + dNumber + "\n" +
                        "  ClassCode: " + ClassCode + "\n" +
                        "  SecCode: " + SecCode + "\n" +
                        "  price: " + dPrice + "\n" +
                        "  balance: " + nBalance + "\n" +
                        "  value: " + dValue + "\n" +
                        "  isSell: " + nIsSell + "\n" +
                        "  status: " + nStatus + "\n" +
                        "  orderDescriptor: " + nOrderDescriptor + "\n");
            }
        };
        JNADll.testCallback(cb);
    }

}

c header QuikWrap.h

#ifdef QUIKWRAP_EXPORTS
#define QUIKWRAP_API extern "C"__declspec(dllexport) 
#else
#define QUIKWRAP_API __declspec(dllimport)
#endif
typedef void (__stdcall *TRANS2QUIK_ORDER_STATUS_CALLBACK) (long nMode, DWORD dwTransID, double dNumber, LPCSTR ClassCode, LPCSTR SecCode, double dPrice, long nBalance, double dValue, long nIsSell, long nStatus, long nOrderDescriptor);
QUIKWRAP_API void __stdcall testCallback(TRANS2QUIK_ORDER_STATUS_CALLBACK cb);

c source QuikWrap.cpp

#include "stdafx.h"
#include "QuikWrap.h"
#include "stdio.h"

QUIKWRAP_API void __stdcall testCallback(TRANS2QUIK_ORDER_STATUS_CALLBACK cb){
    cb(1, 2, 3, "Hi", "Bye", 4.0, 5, 6, 7, 8, 9);
    cb(0, 2, 3, "Hi", "Bye", 4.0, 5, 6, 7, 8, 9);
    return;
}

the only output from callback I got is

testCallback  
  nMode: 1
  dwTransID: 2
  number: 3.0
  ClassCode: Hi
  SecCode: Bye
  price: 4.0
  balance: 5
  value: 6.0
  isSell: 7
  status: 8
  orderDescriptor: 9

then callback fails on exit

Answer 1

The issue is due to a bug in libffi, where the stack alignment is being adjusted according to argument size, which is incorrect behavior for stdcall.