I am trying to implement an Icon handler in C++ based on the example:
The Complete Idiot's Guide to Writing Shell Extensions - Part IX
I have no problem getting the example to work using the example project, but when I try to build it inside of my QT project, my handler is never called.
After installing my DLL, 'ShellExtView' shows it as an "Icon Handler" and everything looks OK in the Registry, as far as I can see.
I took the registration code I have here and used it to register the example shell extension and it worked, so I do not think it is a problem with the way I am registering the shell extension.
Here is my code:
Header File:
#include <Windows.h>
// ATL
#include <atlbase.h>
extern CComModule _Module;
#include <atlcom.h>
#include <atlconv.h>
// Win32
#include <comdef.h>
#include <ShlObj.h>
#define MAX_SUFFIX (32)
#define MY_ID "{E94EFFAC-DBD6-40EF-92FC-460FDEB3684A}"
#define TARGET_ICON_HANDLER_CLASS "txtfile"
const CLSID my_id = {0xE94EFFAC, 0xDBD6, 0x40EF, {0x92, 0XF, 0X46, 0X0F, 0XDE, 0XB3, 0X68, 0X4A}};
const CLSID myLib_id = {0xE94EFFAD, 0xDBD6, 0x40EF, {0x92, 0XF, 0X46, 0X0F, 0XDE, 0XB3, 0X68, 0X4A}};
class CIconShlExt :
public CComObjectRootEx<CComSingleThreadModel>,
public CComCoClass<CIconShlExt, &my_id>,
public IPersistFile,
public IExtractIcon
{
public:
CIconShlExt() : m_haveSuffix(false) { }
BEGIN_COM_MAP(CIconShlExt)
COM_INTERFACE_ENTRY(IPersistFile)
COM_INTERFACE_ENTRY(IExtractIcon)
END_COM_MAP()
DECLARE_NO_REGISTRY()
// IPersistFile
STDMETHODIMP GetClassID( CLSID* pClsId) { return E_NOTIMPL; }
STDMETHODIMP IsDirty() { return E_NOTIMPL; }
STDMETHODIMP Save( LPCOLESTR, BOOL ) { return E_NOTIMPL; }
STDMETHODIMP SaveCompleted( LPCOLESTR ) { return E_NOTIMPL; }
STDMETHODIMP GetCurFile( LPOLESTR* ) { return E_NOTIMPL; }
STDMETHODIMP Load( LPCOLESTR wszFile, DWORD );
// IExtractIcon
STDMETHODIMP GetIconLocation( UINT uFlags, LPTSTR szIconFile, UINT cchMax,
int* piIndex, UINT* pwFlags );
STDMETHODIMP Extract( LPCTSTR pszFile, UINT nIconIndex, HICON* phiconLarge,
HICON* phiconSmall, UINT nIconSize );
WCHAR m_suffix[MAX_SUFFIX];
bool m_haveSuffix;
};
IPersistFile and IExtractIcon methods (that are never called):
#pragma region IPersistFile
STDMETHODIMP CIconShlExt::Load(LPCOLESTR wszFile, DWORD)
{
// I never get here!
return S_OK;
}
#pragma endregion
#pragma region IExtractIcon
STDMETHODIMP CIconShlExt::GetIconLocation(UINT /*uFlags*/, LPTSTR szIconFile, UINT cchMax, int* piIndex, UINT* pwFlags )
{
// I never get here!
// Give it a strange icon so I know it did something
*piIndex = -218;
lstrcpyn(szIconFile, "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\Common7\\IDE\\msenvico.dll", cchMax);
*pwFlags = GIL_PERINSTANCE; // GIL_NOTFILENAME;
return S_OK;
}
STDMETHODIMP CIconShlExt::Extract( LPCTSTR , UINT , HICON*, HICON* , UINT)
{
return S_FALSE;
}
#pragma endregion
The main DLL file:
static DWORD SetRegistryKeyAndValue(HKEY root, const char *key, const char *value, const char *name)
{
HKEY hKey = NULL;
DWORD err;
err = RegCreateKeyEx(root, key, 0, NULL, REG_OPTION_NON_VOLATILE, KEY_WRITE, NULL, &hKey, NULL);
if (err == ERROR_SUCCESS) {
if (name != NULL) {
// Set the specified value of the key.
DWORD cbData = lstrlen(name) * sizeof(*name) +1;
err = RegSetValueEx(hKey, value, 0, REG_SZ, reinterpret_cast<const BYTE *>(name), cbData);
}
RegCloseKey(hKey);
}
return err;
}
static void unregisterHandler()
{
RegDeleteTree(HKEY_CLASSES_ROOT, "CLSID\\" MY_ID);
RegDeleteKey(HKEY_LOCAL_MACHINE, "Software\\Microsoft\\Windows\\CurrentVersion\\Shell Extensions\\Approved\\" MY_ID);
RegDeleteTree(HKEY_CLASSES_ROOT, TARGET_ICON_HANDLER_CLASS "\\ShellEx\\IconHandler");
}
static DWORD registerHandler(const char *dll)
{
DWORD err;
if ((err = SetRegistryKeyAndValue(HKEY_CLASSES_ROOT, "CLSID\\" MY_ID, nullptr, "My icon extension")) != ERROR_SUCCESS)
goto error;
if ((err = SetRegistryKeyAndValue(HKEY_CLASSES_ROOT, "CLSID\\" MY_ID"\\InprocServer32", NULL, dll)) != ERROR_SUCCESS)
goto error;
if ((err = SetRegistryKeyAndValue(HKEY_CLASSES_ROOT, "CLSID\\" MY_ID"\\InprocServer32", "ThreadingModel", "Apartment")) != ERROR_SUCCESS)
goto error;
if ((err = SetRegistryKeyAndValue(HKEY_LOCAL_MACHINE, "Software\\Microsoft\\Windows\\CurrentVersion\\Shell Extensions\\Approved", MY_ID, "My icon extension")) != ERROR_SUCCESS)
goto error;
if ((err = SetRegistryKeyAndValue(HKEY_CLASSES_ROOT, TARGET_ICON_HANDLER_CLASS "\\ShellEx\\IconHandler", nullptr, MY_ID)) != ERROR_SUCCESS)
goto error;
if ((err = SetRegistryKeyAndValue(HKEY_CLASSES_ROOT, TARGET_ICON_HANDLER_CLASS "\\DefaultIcon", nullptr, "%1")) != ERROR_SUCCESS)
goto error;
return err;
error:
unregisterHandler();
return err;
}
BEGIN_OBJECT_MAP(ObjectMap)
OBJECT_ENTRY(my_id, CIconShlExt)
END_OBJECT_MAP()
CComModule _Module;
extern "C"
BOOL WINAPI DllMain(HINSTANCE hInstance, DWORD dwReason, LPVOID /*lpReserved*/)
{
if (dwReason == DLL_PROCESS_ATTACH)
{
_Module.Init(ObjectMap, hInstance, &myLib_id);
DisableThreadLibraryCalls(hInstance);
}
else if (dwReason == DLL_PROCESS_DETACH)
_Module.Term();
return TRUE; // ok
}
/////////////////////////////////////////////////////////////////////////////
// Used to determine whether the DLL can be unloaded by OLE
STDAPI DllCanUnloadNow()
{
return (_Module.GetLockCount()==0) ? S_OK : S_FALSE;
}
/////////////////////////////////////////////////////////////////////////////
// Returns a class factory to create an object of the requested type
STDAPI DllGetClassObject(REFCLSID rclsid, REFIID riid, LPVOID* ppv)
{
return _Module.GetClassObject(rclsid, riid, ppv);
}
STDAPI DllRegisterServer()
{
char dllPath[MAX_PATH];
if (GetModuleFileName(_Module.m_hInst, dllPath, ARRAYSIZE(dllPath)) == 0)
{
HRESULT hr = HRESULT_FROM_WIN32(GetLastError());
return hr;
}
DWORD rtc = registerHandler(dllPath);
if (rtc != ERROR_SUCCESS)
return HRESULT_FROM_WIN32(rtc);
return _Module.RegisterServer(false);
}
STDAPI DllUnregisterServer(void)
{
unregisterHandler();
return _Module.UnregisterServer(false);
}
I found the problem:
I made a mistake when converting the GUID string to a CLSID.
Bad Conversion (the 0XF):
#define MY_ID "{E94EFFAC-DBD6-40EF-92FC-460FDEB3684A}"
const CLSID my_id = {0xE94EFFAC, 0xDBD6, 0x40EF, {0x92, 0XF, 0X46, 0X0F, 0XDE, 0XB3, 0X68, 0X4A}};
Correct conversion:
#define MY_ID "{E94EFFAC-DBD6-40EF-92FC-460FDEB3684A}"
const CLSID my_id = {0xE94EFFAC, 0xDBD6, 0x40EF, {0x92, 0XFC, 0X46, 0X0F, 0XDE, 0XB3, 0X68, 0X4A}};