Can you register a directshow filter without admin?

I would like to register a virtual directshow source device without needing admin. On another post I saw someone reference that you can register COM classes per user account via HKEY_CURRENT_USER instead of HKEY_CLASSES_ROOT or HKEY_LOCAL_MACHINE. This worked, and I could register the class through HKEY_CURRENT_USER\Software\Classes without UAC.

To get the source/filter to appear I need to make a call to IFilterMapper2::RegisterFilter. This fails without UAC privledge. (E_ACCESSDENIED General access denied error.).

Microsoft isn't exactly clear on what the call to RegisterFilter actually does. I know it creates a registry entry under HKEY_CLASSES_ROOT\CLSID\category clsid\Instance\filter clsid but one of the key values is FilterData which is a binary value that should theoretically match https://learn.microsoft.com/en-us/windows/win32/api/strmif/ns-strmif-regfilter2 this struct; but the data doesn't totally line up so they must write other data in there too.

Is there anyway to register the filter on the user account level?

Solution

First you have to register the server:

STDAPI AMovieSetupRegisterServerOverride(CLSID clsServer, LPCWSTR szDescription, LPCWSTR szFileName, LPCWSTR szThreadingModel = L"Both", LPCWSTR szServerType = L"InprocServer32")
{
    TCHAR achTemp[MAX_PATH];

    OLECHAR szCLSID[CHARS_IN_GUID];
    HRESULT hr = StringFromGUID2(clsServer, szCLSID, CHARS_IN_GUID);

    HKEY hkey;
    DWORD d;
    wsprintf(achTemp, TEXT("Software\\Classes\\CLSID\\%ls"), szCLSID);
    LONG lreturn = RegCreateKeyEx(HKEY_CURRENT_USER, (LPCTSTR)achTemp, 0, NULL, 0, KEY_ALL_ACCESS, NULL, &hkey, &d);
    if (ERROR_SUCCESS != lreturn)
    {
        return AmHresultFromWin32(lreturn);
    }

    wsprintf(achTemp, TEXT("%ls"), szDescription);
    lreturn = RegSetValue(hkey, (LPCTSTR)NULL, REG_SZ, achTemp, sizeof(achTemp));
    if (ERROR_SUCCESS != lreturn)
    {
        RegCloseKey(hkey);
        return AmHresultFromWin32(lreturn);
    }
    HKEY hsubkey;

    wsprintf(achTemp, TEXT("%ls"), szServerType);
    lreturn = RegCreateKeyEx(hkey, achTemp, 0, NULL, 0, KEY_ALL_ACCESS, NULL, &hsubkey, &d);
    if (ERROR_SUCCESS != lreturn)
    {
        RegCloseKey(hkey);
        return AmHresultFromWin32(lreturn);
    }
    wsprintf(achTemp, TEXT("%ls"), szFileName);
    lreturn = RegSetValue(hsubkey, (LPCTSTR)NULL, REG_SZ, (LPCTSTR)achTemp, sizeof(TCHAR) * (lstrlen(achTemp) + 1));
    if (ERROR_SUCCESS != lreturn)
    {
        RegCloseKey(hkey);
        RegCloseKey(hsubkey);
        return AmHresultFromWin32(lreturn);
    }
    wsprintf(achTemp, TEXT("%ls"), szThreadingModel);
    lreturn = RegSetValueEx(hsubkey, TEXT("ThreadingModel"), 0L, REG_SZ, (CONST BYTE*)achTemp, sizeof(TCHAR) * (lstrlen(achTemp) + 1));

    RegCloseKey(hkey);
    RegCloseKey(hsubkey);

    ASSERT(SUCCEEDED(hr));
    return HRESULT_FROM_WIN32(lreturn);

}

STDAPI
AMovieSetupUnregisterServerOverride(CLSID clsServer)
{
    OLECHAR szCLSID[CHARS_IN_GUID];
    HRESULT hr = StringFromGUID2(clsServer, szCLSID, CHARS_IN_GUID);
    TCHAR achBuffer[260];
    wsprintf(achBuffer, TEXT("CLSID\\%ls"), szCLSID);
    hr = EliminateSubKey(HKEY_CLASSES_ROOT, achBuffer);
    return NOERROR;
}

This is copied from the winsdk method, but modified to use the Software\Classes registry instead to avoid UAC:

AMovieSetupRegisterServer

Found Here

Then you can register the filter using the following code (note, I didn't finish figuring out exactly how to serialize the filter data so I registered it with UAC, checked what the binary data was for that, and manually fill it out that way. If you want to serialize the filter data yourself you can reverse how they do it in GraphStudioNext (method FilterTemplate::LoadFilterData)

// {ae7416c6-637d-4b8e-970d-24af9a1dd02a}
DEFINE_GUID(CLSID_EXAMPLE_GUID,
    0xae7416c6, 0x637d, 0x4b8e, 0x97, 0x0d, 0x24, 0xaf, 0x9a, 0x1d, 0xd0, 0x2a);

HRESULT RegisterCurrentUser(REGFILTER2 rf)
{
    TCHAR achTemp[MAX_PATH];

    OLECHAR szCLSID[CHARS_IN_GUID];
    HRESULT hr = StringFromGUID2(CLSID_EXAMPLE_GUID, szCLSID, CHARS_IN_GUID);

    HKEY hkey;
    wsprintf(achTemp, TEXT("Software\\Classes\\CLSID\\{860BB310-5D01-11d0-BD3B-00A0C911CE86}\\Instance\\%ls"), szCLSID);
    hr = RegCreateKey(HKEY_CURRENT_USER, (LPCTSTR)achTemp, &hkey);

    wsprintf(achTemp, TEXT("%ls"), szCLSID);
    hr = RegSetValueEx(hkey, TEXT("CLSID"), 0L, REG_SZ, (CONST BYTE*) achTemp, sizeof(TCHAR) * (lstrlen(achTemp) + 1));

    DWORD data[20];
    data[0] = 0x00000002; //Version
    data[1] = 0x00200000; //Merit
    data[2] = 0x00000001; //if v===1, pin#
    data[3] = 0x00000000; //if v==2, pin #
    data[4] = 0x33697030;
    data[5] = 0x08000000;
    data[6] = 0x00000000;
    data[7] = 0x00000001;
    data[8] = 0x00000000;
    data[9] = 0x00000000;
    data[10] = 0x33797430;
    data[11] = 0x00000000;
    data[12] = 0x00000038;
    data[13] = 0x00000048;
    data[14] = 0x73646976;
    data[15] = 0x00100000;
    data[16] = 0xAA000080;
    data[17] = 0x719B3800;
    data[18] = 0x00000000;
    data[19] = 0x00000000;


    hr = RegSetValueEx(hkey, TEXT("FilterData"), 0L, REG_BINARY, (BYTE*)data, sizeof(data));

    wsprintf(achTemp, TEXT("%ls"), L"Example Filter");
    hr = RegSetValueEx(hkey, TEXT("FriendlyName"), 0L, REG_SZ, (CONST BYTE*) achTemp, sizeof(TCHAR) * (lstrlen(achTemp) + 1));
    return hr;
}