Search code examples
javawinapistructurejna

How do I properly map CERT_SELECT_STRUCT in JNA

I am using jna-4.5.1 in my Java Project. This is cryptdlg structure CERT_SELECT_STRUCT I want to replicate.

typedef struct tagCSSA {
  DWORD           dwSize;
  HWND            hwndParent;
  HINSTANCE       hInstance;
  LPCSTR          pTemplateName;
  DWORD           dwFlags;
  LPCSTR          szTitle;
  DWORD           cCertStore;
  HCERTSTORE      *arrayCertStore;
  LPCSTR          szPurposeOid;
  DWORD           cCertContext;
  PCCERT_CONTEXT  *arrayCertContext;
  LPARAM          lCustData;
  PFNCMHOOKPROC   pfnHook;
  PFNCMFILTERPROC pfnFilter;
  LPCSTR          szHelpFileName;
  DWORD           dwHelpId;
  HCRYPTPROV      hprov;
} CERT_SELECT_STRUCT_A, *PCERT_SELECT_STRUCT_A;

Sample Java code for my project.

public class Crypto {
    public interface Cryptdlg extends Library {
        Cryptdlg INSTANCE = (Cryptdlg) Native.loadLibrary("Cryptdlg", Cryptdlg.class, W32APIOptions.DEFAULT_OPTIONS);

        public boolean CertSelectCertificate(CERT_SELECT_STRUCT pCertSelectInfo);

        public static class CERT_SELECT_STRUCT extends Structure {

            private static final List<String> fieldOrder = createFieldsOrder("dwSize", "hwndParent", "hInstance",
                    "pTemplateName", "dwFlags", "szTitle", "cCertStore", "arrayCertStore", "szPurposeOid",
                    "cCertContext", "arrayCertContext", "lCustData", "pfnHook", "pfnFilter", "szHelpFileName",
                    "dwHelpId", "hprov");

            public static class ByReference extends CERT_SELECT_STRUCT implements Structure.ByReference {
            }

            public int dwSize = size();
            public HWND hwndParent;
            public HINSTANCE hInstance;
            public String pTemplateName;
            public int dwFlags;
            public String szTitle;
            public int cCertStore;
            public Pointer arrayCertStore;
            public String szPurposeOid;
            public int cCertContext;
            public Pointer arrayCertContext;
            public WinDef.LPARAM lCustData;
            public Pointer pfnHook = null;
            public Pointer pfnFilter = null;
            public String szHelpFileName;
            public int dwHelpId;
            public HCRYPTPROV hprov;

            public CERT_SELECT_STRUCT() {
                super();
            }

            public WinCrypt.CERT_CONTEXT[] getArrayCertContext() {
                WinCrypt.CERT_CONTEXT[] elements = new WinCrypt.CERT_CONTEXT[cCertContext];
                for (int i = 0; i < elements.length; i++) {
                    elements[i] = (WinCrypt.CERT_CONTEXT) Structure.newInstance(WinCrypt.CERT_CONTEXT.class,
                            arrayCertContext.getPointer(i * Native.POINTER_SIZE));
                    elements[i].read();
                }
                return elements;
            }

            public void setArrayCertContext(WinCrypt.CERT_CONTEXT[] arrayCertContexts) {
                if (arrayCertContexts == null || arrayCertContexts.length == 0) {
                    arrayCertContext = null;
                    cCertContext = 0;
                } else {
                    cCertContext = arrayCertContexts.length;
                    Memory mem = new Memory(Native.POINTER_SIZE * arrayCertContexts.length);
                    for (int i = 0; i < arrayCertContexts.length; i++) {
                        mem.setPointer(i * Native.POINTER_SIZE, arrayCertContexts[i].getPointer());
                    }
                    arrayCertContext = mem;
                }
            }

            public void setArrayCertStore(WinCrypt.HCERTSTORE[] stores) {
                if (stores == null || stores.length == 0) {
                    arrayCertStore = null;
                    cCertStore = 0;
                } else {
                    Memory mem = new Memory(Native.POINTER_SIZE * stores.length);
                    for (int i = 0; i < stores.length; i++) {
                        mem.setPointer(i * Native.POINTER_SIZE, stores[i].getPointer());
                    }
                    cCertStore = stores.length;
                    arrayCertStore = mem;
                }
            }

            public WinCrypt.HCERTSTORE[] getArrayCertStore() {
                if (arrayCertStore == null || cCertStore == 0) {
                    return new WinCrypt.HCERTSTORE[0];
                } else {
                    WinCrypt.HCERTSTORE[] result = new WinCrypt.HCERTSTORE[cCertStore];
                    for (int i = 0; i < result.length; i++) {
                        result[i] = new WinCrypt.HCERTSTORE(arrayCertStore.getPointer(i * Native.POINTER_SIZE));
                    }
                    return result;
                }
            }

            @Override
            protected List<String> getFieldOrder() {
                return fieldOrder;
            }
        }
    }

    public void CertSelect() {
       Cryptdlg cryptdlg = Cryptdlg.INSTANCE;

     ...// parentHwnd and hCertStore are initalized and passed to this method
     Cryptdlg.CERT_SELECT_STRUCT certSel = new Cryptdlg.CERT_SELECT_STRUCT();
     WinCrypt.CERT_CONTEXT[] pContexts = new WinCrypt.CERT_CONTEXT[1];
     certSel.hwndParent = parentHwnd;
     certSel.szTitle = "title";
     certSel.cCertStore = 1;
     certSel.setArrayCertStore(new WinCrypt.HCERTSTORE[] {hCertStore});
     pCertSelectInfo.cCertContext = 1;
     pContexts[0] = new WinCrypt.CERT_CONTEXT.ByReference();
     certSel.setArrayCertContext(pContexts);
     cryptdlg.CertSelectCertificate(certSel); //line 60

    ...
    }
}

When I call this method I get "java.lang.Error: Invalid memory access" at the dll call cryptdlg.CertSelectCertificate(certSel) at line 60 above.

java.lang.Error: Invalid memory access
at com.sun.jna.Native.invokeInt(Native Method)
    at com.sun.jna.Function.invoke(Function.java:419)
    at com.sun.jna.Function.invoke(Function.java:354)
    at com.sun.jna.Library$Handler.invoke(Library.java:244)
    at com.sun.proxy.$Proxy13.CertSelect(Unknown Source)
    at com.project.Crypto.CertSelect(Crypto.java:60)

I am not sure why we are getting the exception. I followed the example mentioned here.

[UPDATE]

For what its worth, When I modify the type of "setArrayCertStore" from Pointer to HCERTSTORE[] I am not getting any exception but no certificate are getting pulled.No Cert

It makes me think if arrayCertStore is initalized correctly or not.

WinCrypt.HCERTSTORE[] cStoreArray = new WinCrypt.HCERTSTORE[1];
pCertSelectInfo.cCertStore = 1;
cStoreArray[0] = hCertStore;
pCertSelectInfo.arrayCertStore = cStoreArray;

And the structure definition is changed as follows

public WinCrypt.HCERTSTORE[] arrayCertStore;

And HCRYPTPROV is defined as

    public static class HCRYPTPROV extends BaseTSD.ULONG_PTR {

      public HCRYPTPROV() {}
      public HCRYPTPROV(long value) {
      super(value);
      }
    }

==================================

[EDIT] After discussion with Daniel and other people. Here is the updated code which works

public class Crypto {
    public interface Cryptdlg extends Library {
        Cryptdlg INSTANCE = (Cryptdlg) Native.loadLibrary("Cryptdlg", Cryptdlg.class, W32APIOptions.DEFAULT_OPTIONS);

        public boolean CertSelectCertificate(CERT_SELECT_STRUCT pCertSelectInfo);

        public static class CERT_SELECT_STRUCT extends Structure {

            private static final List<String> fieldOrder = createFieldsOrder("dwSize", "hwndParent", "hInstance",
                    "pTemplateName", "dwFlags", "szTitle", "cCertStore", "arrayCertStore", "szPurposeOid",
                    "cCertContext", "arrayCertContext", "lCustData", "pfnHook", "pfnFilter", "szHelpFileName",
                    "dwHelpId", "hprov");

            public static class ByReference extends CERT_SELECT_STRUCT implements Structure.ByReference {
            }

            public int dwSize;
            public HWND hwndParent;
            public HINSTANCE hInstance;
            public String pTemplateName;
            public int dwFlags;
            public String szTitle;
            public int cCertStore;
            public Pointer arrayCertStore;
            public String szPurposeOid;
            public int cCertContext;
            public Pointer arrayCertContext;
            public WinDef.LPARAM lCustData;
            public Pointer pfnHook = null;
            public Pointer pfnFilter = null;
            public String szHelpFileName;
            public int dwHelpId;
            public HCRYPTPROV hprov;

            public CERT_SELECT_STRUCT() {
                super();
            }

            public WinCrypt.CERT_CONTEXT[] getArrayCertContext() {
                WinCrypt.CERT_CONTEXT[] elements = new WinCrypt.CERT_CONTEXT[cCertContext];
                for (int i = 0; i < elements.length; i++) {
                    elements[i] = (WinCrypt.CERT_CONTEXT) Structure.newInstance(WinCrypt.CERT_CONTEXT.class,
                            arrayCertContext.getPointer(i * Native.POINTER_SIZE));
                    elements[i].read();
                }
                return elements;
            }

            public void setArrayCertContext(WinCrypt.CERT_CONTEXT[] arrayCertContexts) {
                if (arrayCertContexts == null || arrayCertContexts.length == 0) {
                    arrayCertContext = null;
                    cCertContext = 0;
                } else {
                    cCertContext = arrayCertContexts.length;
                    Memory mem = new Memory(Native.POINTER_SIZE * arrayCertContexts.length);
                    for (int i = 0; i < arrayCertContexts.length; i++) {
                        mem.setPointer(i * Native.POINTER_SIZE, arrayCertContexts[i].getPointer());
                    }
                    arrayCertContext = mem;
                }
            }

            public void setArrayCertStore(WinCrypt.HCERTSTORE[] stores) {
                if (stores == null || stores.length == 0) {
                    arrayCertStore = null;
                    cCertStore = 0;
                } else {
                    Memory mem = new Memory(Native.POINTER_SIZE * stores.length);
                    for (int i = 0; i < stores.length; i++) {
                        mem.setPointer(i * Native.POINTER_SIZE, stores[i].getPointer());
                    }
                    cCertStore = stores.length;
                    arrayCertStore = mem;
                }
            }

            public WinCrypt.HCERTSTORE[] getArrayCertStore() {
                if (arrayCertStore == null || cCertStore == 0) {
                    return new WinCrypt.HCERTSTORE[0];
                } else {
                    WinCrypt.HCERTSTORE[] result = new WinCrypt.HCERTSTORE[cCertStore];
                    for (int i = 0; i < result.length; i++) {
                        result[i] = new WinCrypt.HCERTSTORE(arrayCertStore.getPointer(i * Native.POINTER_SIZE));
                    }
                    return result;
                }
            }

            @Override
            public void write() {
                this.dwSize = size();
                super.write();
            }
            
            @Override
            protected List<String> getFieldOrder() {
                return fieldOrder;
            }
        }
    }

    public void CertSelect() {
     Cryptdlg cryptdlg = Cryptdlg.INSTANCE;
     Cryptdlg.CERT_SELECT_STRUCT certSel = new Cryptdlg.CERT_SELECT_STRUCT();
     certSel.hwndParent = parentHwnd;
     certSel.szTitle = "title";
     certSel.cCertStore = 1;
     certSel.setArrayCertStore(new WinCrypt.HCERTSTORE[] {hCertStore});
     pCertSelectInfo.cCertContext = 1;
     pCertSelectInfo.arrayCertContext = new Memory(Native.POINTER_SIZE);
     pCertSelectInfo.arrayCertContext.setPointer(0, Pointer.NULL);
     cryptdlg.CertSelectCertificate(certSel);

    ...
    }
}
Solution

  • There are a few potential problem areas.

    First, you are mixing the ANSI and Unicode versions.

    The CertSelectCertificate() function has two variants, one ending in A and one in W. The -A functions are the ANSI (8-bit characters) while the -W variants are for Unicode/Wide-string (16 bit characters). The main difference in the methods is the number of characters in the strings in the CERT_SELECT_STRUCT structure., which similarly has two variants, CERT_SELECT_STRUCT_A and CERT_SELECT_STRUCT_W.

    JNA automatically maps you to the correct version (the -W version in almost all cases in modern operating systems) using its default type mapper. You should probably add that type mapper explicitly in your library loading using the default options for Win32 libraries:

    Cryptdlg INSTANCE = (Cryptdlg) Native.loadLibrary("Cryptdlg", Cryptdlg.class,
    W32APIOptions.DEFAULT_OPTIONS);

    However, you have an explicit call to use the -A type mapper in your CERT_SELECT_STRUCT() constructor:

    public CERT_SELECT_STRUCT() {
  super(W32APITypeMapper.ASCII);
}

    This forces it to use the -A version of the structure, which only has 8 bits per character in its strings. You should just call super().

    A second possibility, while not obvious from the docs, is that the first element, dwSize, is supposed to be the size of the structure. You should put that in your constructor:

    this.dwSize = this.size();

    A third possibility (and the most likely cause, if I were to guess) is in the line when you set the contents of the arrayCertContext field. It is documented as:

    A pointer to an array of CERT_CONTEXT structures.

    You define the array on the Java side as a structure (which has its own pointer) and manually set it into memory, but instead of populating the CERT_CONTEXT structure you put a Pointer there:

    pContexts[0] = new WinCrypt.CERT_CONTEXT.ByReference();

    That ends up filling up the first 8 bytes of the "structure" with the pointer address you just created, the first 4 bytes of which are assigned to dwCertEncodingType and the next 4 bytes (plus 4 null bytes) go to the pointer value of a ByteByReference() field.

    Also, easier than your allocation of memory, allocation of arrays of structures could be done like this:

    WinCrypt.CERT_CONTEXT[] pContextArray = 
    (WinCrypt.CERT_CONTEXT[]) new WinCrypt.CERT_CONTEXT().toArray(1);

    As an aside, you might find structure definitions more streamlined in JNA 5.x (currently 5.6.0) which include a @FieldOrder annotation as a preferred method of creating the field order list.