Mouse and Keyboard system Global Hook

Question

I want to set a global system hook for keyboard and mouse in C++ using a DLL. My program is working but not exactly as I need. For example the program will not hook mouse events on other windows or popup menu, or modal dialogs. I want to hook the mouse and keyboard events anywhere in other windows, programs, ect.., all system mouse and keyboard events. What is wrong with my code what to change to get it to work as I need, I cannot find myself what is wrong, I'm doing as is described in documentation, I need help.

I'm coding in C++ Builder using VCL

DLL:

#include <vcl.h>
#include <windows.h>
#include "main.h"
#pragma hdrstop
#pragma argsused

typedef struct _HOOKSTRUCT
{
    int nType;
    HOOKPROC hkprc;
    HHOOK hhook;
    bool bInstalled;
} HOOKSTRUCT;

static HOOKSTRUCT hook[2];
HINSTANCE hDLL;
int nThreadCode;
bool bInit;
TForm1 *form;
extern "C" __stdcall __declspec(dllexport) LRESULT CALLBACK KeyboardProc(int nCode, WPARAM wParam, LPARAM lParam);
extern "C" __stdcall __declspec(dllexport) LRESULT CALLBACK MouseProc(int nCode, WPARAM wParam, LPARAM lParam);
//---------------------------------------------------------------------------
int WINAPI DllEntryPoint(HINSTANCE hinst, unsigned long reason, void* lpReserved)
{
        hDLL = hinst;
        bInit = false;
        return 1;
}
//---------------------------------------------------------------------------
extern "C" __stdcall __declspec(dllexport) void Init()
{
        hook[0].nType = WH_KEYBOARD;
        hook[0].hkprc = (HOOKPROC)KeyboardProc;
        hook[0].bInstalled = false;
        hook[1].nType = WH_MOUSE;
        hook[1].hkprc = (HOOKPROC)MouseProc;
        hook[1].bInstalled = false;
        bInit = true;
}
//---------------------------------------------------------------------------
extern "C" __stdcall __declspec(dllexport) bool SetHook(int nHook)
{
        if(!bInit)
        {
                Init();
        }
        hook[nHook].hhook = NULL;
        hook[nHook].hhook = SetWindowsHookEx(hook[nHook].nType,
                (HOOKPROC)hook[nHook].hkprc, hDLL, 0);
        if(hook[nHook].hhook != NULL)
        {
                MessageBox(NULL, "Setup hook successful", "Information", MB_OK);
        }
        return true;
}
//---------------------------------------------------------------------------
extern "C" __stdcall __declspec(dllexport) void UnsetHook(int nHook)
{
        UnhookWindowsHookEx(hook[nHook].hhook);
}
//---------------------------------------------------------------------------
extern "C" __stdcall __declspec(dllexport) LRESULT CALLBACK KeyboardProc(int nCode, WPARAM wParam, LPARAM lParam)
{
        if(((DWORD)lParam&0x40000000) && (HC_ACTION==nCode))
        {
                if(form != NULL)
                {
                        form->Memo1->Lines->Add("Key Pressed");
                }
        }
        return CallNextHookEx(hook[0].hhook, nCode, wParam, lParam);
}
//---------------------------------------------------------------------------
extern "C" __stdcall __declspec(dllexport) LRESULT CALLBACK MouseProc(int nCode, WPARAM wParam, LPARAM lParam)
{
        char str[128];
        PMOUSEHOOKSTRUCT Info = (PMOUSEHOOKSTRUCT)lParam;
        wsprintf(str, "The mouse message at X: %d,Y: %d", Info->pt.x, Info->pt.y);
        form->Memo2->Lines->Add(str);
        return CallNextHookEx(hook[1].hhook, nCode, wParam, lParam);
}
//---------------------------------------------------------------------------
extern "C" __stdcall __declspec(dllexport) void SetControl(TForm1 *Object, int nApp)
{
        form =  Object;
        nThreadCode = nApp;
}

Program:

#include <vcl.h>
#pragma hdrstop

#include "main.h"
//---------------------------------------------------------------------------
#pragma package(smart_init)
#pragma resource "*.dfm"
TForm1 *Form1;
HINSTANCE hInst;
typedef bool (*install)(int);
typedef void (*uninstall)(int);
typedef void (*passself)(TForm1 *, int);
install InstallHook;
uninstall UninstallHook;
passself PassThis;
//---------------------------------------------------------------------------
__fastcall TForm1::TForm1(TComponent* Owner)
        : TForm(Owner)
{
        N2->Enabled = true;
        N3->Enabled = true;
        N4->Enabled = false;
        N5->Enabled = false;
}
//---------------------------------------------------------------------------
void __fastcall TForm1::FormCreate(TObject *Sender)
{
        Memo1->Clear();
        Memo2->Clear();
        hInst = LoadLibrary("ch49dll.dll");
        if(hInst == NULL)
        {
                ShowMessage("Load DLL error " + AnsiString(GetLastError()));
                return;
        }
        InstallHook = (install)GetProcAddress(hInst, "SetHook");
        if(InstallHook == NULL)
        {
                ShowMessage("Get SetHook ProcAddress Error");
                return;
        }
        UninstallHook = (uninstall)GetProcAddress(hInst, "UnsetHook");
        if(UninstallHook == NULL)
        {
                ShowMessage("Get UnsetHook ProcAddress Error");
                return;
        }
        PassThis = (passself)GetProcAddress(hInst, "SetControl");
        if(PassThis == NULL)
        {
                ShowMessage("Get SetControl ProcAddress Error");
                return;
        }
        PassThis(this, 0);
}
//---------------------------------------------------------------------------
void __fastcall TForm1::FormDestroy(TObject *Sender)
{
        FreeLibrary(hInst);
}
//---------------------------------------------------------------------------
void __fastcall TForm1::N2Click(TObject *Sender)
{
        InstallHook(0);
        N2->Enabled = false;
        N4->Enabled = true;
}
//---------------------------------------------------------------------------
void __fastcall TForm1::N3Click(TObject *Sender)
{
        InstallHook(1);
        N3->Enabled = false;
        N5->Enabled = true;
}
//---------------------------------------------------------------------------
void __fastcall TForm1::N4Click(TObject *Sender)
{
        UninstallHook(0);
        N4->Enabled = false;
        N2->Enabled = true;
}
//---------------------------------------------------------------------------
void __fastcall TForm1::N5Click(TObject *Sender)
{
        UninstallHook(1);
        N5->Enabled = false;
        N3->Enabled = true;
}
//---------------------------------------------------------------------------
void __fastcall TForm1::N7Click(TObject *Sender)
{
        Close();
}
//---------------------------------------------------------------------------

Program header:

#ifndef mainH
#define mainH
//---------------------------------------------------------------------------
#include <Classes.hpp>
#include <Controls.hpp>
#include <StdCtrls.hpp>
#include <Forms.hpp>
#include <Menus.hpp>
//---------------------------------------------------------------------------
class TForm1 : public TForm
{
__published:    // IDE-managed Components
        TMemo *Memo1;
        TMemo *Memo2;
        TMainMenu *MainMenu1;
        TMenuItem *N1;
        TMenuItem *N2;
        TMenuItem *N3;
        TMenuItem *N4;
        TMenuItem *N5;
        TLabel *Label1;
        TLabel *Label2;
        TMenuItem *N6;
        TMenuItem *N7;
        void __fastcall FormCreate(TObject *Sender);
        void __fastcall FormDestroy(TObject *Sender);
        void __fastcall N2Click(TObject *Sender);
        void __fastcall N3Click(TObject *Sender);
        void __fastcall N4Click(TObject *Sender);
        void __fastcall N5Click(TObject *Sender);
        void __fastcall N7Click(TObject *Sender);
private:    // User declarations
public:     // User declarations
        __fastcall TForm1(TComponent* Owner);
};
//---------------------------------------------------------------------------
extern PACKAGE TForm1 *Form1;
//---------------------------------------------------------------------------
#endif

So the program work and can hook external events, but for some dialog windows , my program will not hook the mouse events. for example when I right click the icon from taskbar Notification Area , a popup will display and if I enter mouse cursor in the popup window area then my program will not record those mouse move events. Other case is when I click About program menu in other programs, or About menu, then a window will display and if I move my cursor inside the area of that window then my program will not hook events of mouse and keyboard, why ? how to make it to hook anywhere for any window

Answer 1

On Win 7 you can only hook messages from programs with the same or lower privileges. I.e. if your own program runs with standard privileges it will never see any Mouse or Keyboard messages targeted at a process with e.g. admin privileges. – iamjoosy

Answer 2

Two very obvious problems:

1. The DLL will be injected into all other processes. You cannot expect to call VCL methods from a DLL that has been hooked into another process. You'll have to find another way to report diagnostics.
2. Because you are using a global hook, you will need to deal with 32/64 bit issues. You would need to produce both 32 and 64 bit versions of the DLL, and set hooks to them from 32 and 64 bit processes.

If you can use a low-level hook you should consider taking that option. The advantage is that there is no injection with a low-level hook. You don't need to create a DLL at all.

Answer 3

A few comments, hopefully will solve your problems: Use the "Low Level" hooks (WH_MOUSE_LL, WH_KEYBOARD_LL) because they are guaranteed to be only global. It feels to me that your application got only a "local" (thread) hook.

I have bad experiences with CBuilder created DLLs when using together with hooks. ... ie: the same problem exactly as you describe. I'd recommend to recompile your DLL with a Microsoft compiler (and provide a .DEF file to use).

There are chances that you will need to place the DLL in a place identified as system directory.

You don't need to place the SetHook in the DLL, that can be done from the "main" application too.