SetWindowLongPtr returning ERROR_ACCESS_DENIED

Question

I am still struggling with hooks.

My goal is:

• Set a hook in notepad.exe
• Subclass it (my final goal is to subclass the Edit class and show the content in my own window)

Disclaimer: I know there are easier ways to get the text/content from notepad, but this is a way for me to learn C, winapi, Subclassing and hooks.

My problem is that the SetWindowLongPtr always return ERROR_ACCESS_DENIED error (code 5).

22th of May 2013: This has been fixed! The problem was that SetWindowLongPtr was in the wrong place. It must be inside the GetMsgProc function.

The question became a bit long and messy, so I re-wrote the question (with updated code)

The problem now is that GetMsgProc is NOT called when target is notepad.exe. If I change target to simple.exe, the GetMsgProc gets called and works!

(Simple.exe is just a simple GUI):

The code looks like this:

exe.cpp

#include <windows.h>
#include "Resource.h"
#include <stdlib.h>
#include "stdafx.h"
#include <strsafe.h>

#include "C:\Users\Kristensen\Documents\Visual Studio 2012\Projects\Win32D\dll\dllHeader.h"

//---------------------------------------------------------------------------
HWND hWnd;

LRESULT CALLBACK DlgProc(HWND hWnd, UINT Msg, WPARAM wParam, LPARAM lParam);
//---------------------------------------------------------------------------
INT WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
                   LPSTR lpCmdLine, int nCmdShow)
{
    DialogBox(hInstance, MAKEINTRESOURCE(IDD_DLGFIRST),
        hWnd, reinterpret_cast<DLGPROC>(DlgProc));

    return FALSE;
}
//---------------------------------------------------------------------------
LRESULT CALLBACK DlgProc(HWND hWndDlg, UINT Msg, WPARAM wParam, LPARAM lParam)
{
    switch(Msg)
    {
    case WM_INITDIALOG:
        return TRUE;

    case WM_COMMAND:
        switch(wParam)
        {
        case IDOK:
            hookNotepad();
            return TRUE;
        case IDCANCEL:
            removeHook();
            EndDialog(hWndDlg, 0);
        }
        break;
    }

    return FALSE;
}
//---------------------------------------------------------------------------

dllHeader.h

    #ifdef DLLAPI
    #else
    #define DLLAPI extern "C" __declspec(dllimport)
    #endif
    DLLAPI bool hookNotepad();
    DLLAPI bool removeHook();

dll.cpp:

#include "stdafx.h"
#include <windows.h>
#define DLLAPI extern "C" __declspec(dllexport)
#include "dllHeader.h"

// shared variables
#pragma data_seg("Shared")
HHOOK g_hHook = NULL; // Hook for Notepad 
HWND npHWND = NULL; // Notepad handle
#pragma data_seg()
#pragma comment(linker, "/section:Shared,rws")

// Forward references
LRESULT CALLBACK GetMsgProc(int nCode, WPARAM wParam, LPARAM lParam) ;
LRESULT CALLBACK NewWndProc(HWND Hwnd, UINT Message, WPARAM wParam, LPARAM lParam);
//LRESULT CALLBACK CBTProc(int nCode, WPARAM wParam, LPARAM lParam) ;


LONG OldWndProc; 
DWORD pid;
HINSTANCE g_hInstDll = NULL; // DllMain entry (DLL_PROCESS_ATTACH)
DWORD npThreadId = NULL; // Notepad thread ID

LRESULT CALLBACK GetMsgProc(int nCode, WPARAM wParam, LPARAM lParam) //Testing with CBTProc - same issues as with GetMsgProc.
{
    //If I hook notepad.exe, I never get called. (silence)

    //If I hook simple.exe, I get called (Beep beep!)

    // make some noise
    static DWORD dwTickKeep = 0;
    if ((GetTickCount()-dwTickKeep)>300)
    {   dwTickKeep = GetTickCount();
    Beep(2000, 100);
    }

    //Subclassing......
    //For simple.exe: (working)
    //HWND hwndEdit = ::FindWindowEx(npHWND,NULL,TEXT("WindowsForms10.RichEdit20W.app.0.2bf8098_r14_ad1"), NULL);
    //For notepad.exe: (not working)
    HWND hwndEdit = ::FindWindowEx(npHWND,NULL,TEXT("Edit"), NULL); 

    if (hwndEdit)
    {
        //Subclass it
        OldWndProc = GetWindowLongPtr(hwndEdit, GWLP_WNDPROC);
        SetWindowLongPtr(hwndEdit, GWL_WNDPROC, (LONG_PTR)NewWndProc);
    }
    return(CallNextHookEx(g_hHook, nCode, wParam, lParam));
}

BOOL APIENTRY DllMain( HMODULE hModule,  DWORD  ul_reason_for_call,   LPVOID lpReserved  )
{
    switch (ul_reason_for_call)
    {
    case DLL_PROCESS_ATTACH:
        g_hInstDll = hModule;
        break;

    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
    case DLL_PROCESS_DETACH:
        break;
    }
    return TRUE;
}

bool hookNotepad ()
{
    // If target is running
    // if (npHWND = FindWindow(NULL, TEXT("simpleGUI")))
    if (npHWND = FindWindow(TEXT("Notepad"), NULL))
    {
        // Finds the ThreadID for target. We use this in SetWindowsHookEx   
        npThreadId = GetWindowThreadProcessId(npHWND, &pid); 

        // Sets the hook in target
        g_hHook = SetWindowsHookEx(WH_GETMESSAGE, GetMsgProc, g_hInstDll, npThreadId); 
        //g_hHook = SetWindowsHookEx(WH_CBT, CBTProc, g_hInstDll, npThreadId); 

        // If the hook succesed
        if (g_hHook) 
        { 
            ////Add a menu in the notepad.exe, but not relevant for subclassing notepads edit class...
            //HMENU hCurrent = GetMenu(npHWND); //Get the CURRENT menu of the window.
            //HMENU hNew = CreateMenu(); //Create a new one. 
            //AppendMenu(hCurrent, MF_STRING | MF_POPUP, (unsigned int)hNew, TEXT("myMenu")); 
            //AppendMenu(hNew, MF_STRING, 2000, L"myButton"); //2000 is the ID of the new button. 
            //DrawMenuBar(npHWND); //redraw the Menu.

            //Force a msg to the messagequeue, so that the hook function(GetMsgProc) gets called
            PostThreadMessage(npThreadId, WM_NULL, 0, 0);
            return 1;
        }
        return 0;
    }
    else
        //Notepad is not running
        return 0;
}

bool removeHook()
{
    // Removes the hook
    if (g_hHook != NULL)
    {
        UnhookWindowsHookEx(g_hHook);
        g_hHook = NULL;
    }
    return 0;
}



LRESULT CALLBACK NewWndProc(HWND Hwnd, UINT Message, WPARAM wParam, LPARAM lParam)
{ 
    //We should come here and should be able to read the text from the Edit class...
    return CallWindowProc((WNDPROC)OldWndProc, Hwnd, Message, wParam, lParam); 
}

Any hints, comments or tips are highly appreciate...

Answer 1

There are 3 problems in your code:

(1) The variable HWND npHWND is meant to be shared between the host-exe and the notepad-exe so it must be placed inside the share data segment block. This value is presently evaluated inside the 'hookNotepad' call and only exist in the host-exe only. This problem resulted in the npHWND handle being null in the notepad-exe and so the SetWindowLongPtr call failed.

(2) There are 2 SetWindowLongPtr calls, one of them is wrong. The one inside GetMsgProc is correct because it will be executed inside the notepad-exe context when the hook is installed. Remove the other wrong one inside hookNotepad.

(3) Even if (1) and (2) are resolved, the final behaviour of SetWindowLongPtr may not be what you expected because the main UI interactive element of the notepad-exe is the embedded edit control and not the main frame window. You should enumerate the child windows of the notepad-frame and sub-class the only one child window with Edit class.

Edit #1 - Add sound indicator code to check activity ------------------------------------

Add this block of code inside the GetMsgProc

// make some noise
static DWORD dwTickKeep = 0;
if ((GetTickCount()-dwTickKeep)>300)
{   dwTickKeep = GetTickCount();
    Beep(2000, 100);
}

Answer 2

You have to do it in the context of the hooked process. Your hookNotepad() function runs in a different process and thus your WndProc() function is in a different address space.