detours hooked CreateFile function triggers stack overflow

Question

When I was trying to use Detours in order to hook CreateFile, when my hooked function is called I get a stack overflow error. I am trying to write the filename to a file and then call the original, but it fails on the fopen call with a stack overflow error. I am injecting the dll via CreateRemoteThread call. Is there some special stack allocation we have to do in the target process. I am fairly new to Windows development and detours, but I know C/C++ fairly well but by no means an expert.

#include "stdafx.h"
#include "detours.h"
#include <cstdio>
#include <ws2tcpip.h>
#include <windows.h>
#include <stdio.h>

#pragma comment(lib, "detours.lib")
//#pragma comment(lib, "detoured.lib")
#pragma comment(lib, "ws2_32.lib")


HANDLE (WINAPI *oldCreate)(LPCTSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, DWORD,DWORD, HANDLE ) = CreateFile;

HANDLE WINAPI myCreate(LPCTSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, DWORD,DWORD,HANDLE); 


INT APIENTRY DllMain(HMODULE hDLL, DWORD Reason, LPVOID Reserved)
{
    switch(Reason)
    {
        case DLL_PROCESS_ATTACH:
    DetourTransactionBegin();
            DetourUpdateThread(GetCurrentThread());
            DetourAttach(&(PVOID&)oldCreate, myCreate);
            DetourTransactionCommit();
            break;

    case DLL_PROCESS_DETACH:
    case DLL_THREAD_ATTACH:
    case DLL_THREAD_DETACH:
        break;
    }
    return TRUE;
}

HANDLE WINAPI myCreate(LPCTSTR lpFileName , DWORD dwDesiredAccess, DWORD dwShareMode , LPSECURITY_ATTRIBUTES lpSecurityAttributes  , DWORD dwCreationDisposition ,DWORD dwFlagsAndAttributes, HANDLE hTemplateFile)
{
    int x= 3;
    FILE *file = fopen("C:\\test.txt", "a+");

    fprintf(file, "%s \n", lpFileName);
    fclose(file);
    return oldCreate(lpFileName,dwDesiredAccess,dwShareMode,lpSecurityAttributes,dwCreationDisposition,dwFlagsAndAttributes,hTemplateFile);

}


extern "C" __declspec(dllexport) void dummy(void){`enter code here`
    return;
}

Here is the injector Code I am using

Also, here is the injector code I am using

#include "stdafx.h"
#include <windows.h>
#include <stdio.h>
#include "detours.h"

#pragma comment (lib, "detours.lib")


#define MAX_COMBINED 8192

BOOL SetPrivilege(
    HANDLE hToken,          // token handle
    LPCTSTR Privilege,      // Privilege to enable/disable
    BOOL bEnablePrivilege   // TRUE to enable.  FALSE to disable
    );

LPTSTR GetArguments(void)
{
    LPWSTR *szArglist = NULL;
    int nArgs;
    LPWSTR  wbuf = NULL;

    wbuf = new WCHAR[MAX_COMBINED];

    if (wbuf == NULL)
        return NULL;

    memset(wbuf, 0, MAX_COMBINED*sizeof(WCHAR));
    szArglist = CommandLineToArgvW(GetCommandLineW(), &nArgs);
    if(NULL == szArglist)
    {
        return NULL;
    }
    else {
        for(int i=2; i<nArgs; i++) {
            wcscat_s(wbuf, MAX_COMBINED, szArglist[i]);
            wcscat_s(wbuf, MAX_COMBINED, L" ");
        }
    }
    LocalFree(szArglist);

#ifdef _UNICODE
    return wbuf;
#else
    LPSTR abuf = new CHAR[MAX_COMBINED];

    if (abuf == NULL) 
        return NULL;

    memset(abuf, 0, MAX_COMBINED);
    WideCharToMultiByte(CP_ACP, 0, wbuf, -1, abuf, MAX_COMBINED, NULL, NULL);

    delete[] wbuf;
    return abuf;
#endif
}

int _tmain(int argc, _TCHAR* argv[])
{
        HANDLE hToken;
        if(argc < 2)
        {
            printf("pass just pid]\n");
            return 0;
        }
        char* DirPath = new char[MAX_PATH];
        char* FullPath = new char[MAX_PATH];
        GetCurrentDirectoryA(MAX_PATH, (LPSTR)DirPath);
        sprintf_s(FullPath, MAX_PATH, "%s\\injector3.dll", DirPath);
        printf("FullPath %s \n",FullPath);
        if(!OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, FALSE, &hToken))
        {
            if (GetLastError() == ERROR_NO_TOKEN)
            {
                if (!ImpersonateSelf(SecurityImpersonation))
                return 1;

                if(!OpenThreadToken(GetCurrentThread(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, FALSE, &hToken)){
                    printf("OpenThreadToken\n");
                    return 1;
                }
            }
            else
                return 1;
        }

    // enable SeDebugPrivilege
        if(!SetPrivilege(hToken, SE_DEBUG_NAME, TRUE))
        {
            printf("SetPrivilege");

            // close token handle
            CloseHandle(hToken);

            // indicate failure
            return 2;
        }

        HANDLE hProcess = OpenProcess(PROCESS_CREATE_THREAD    | PROCESS_VM_OPERATION    |
            PROCESS_VM_WRITE, FALSE, _wtoi(argv[1]));
        if(hProcess == NULL)
        {
            DWORD x =  GetLastError();
            printf("HANDLE TO PROCESS FAILED on PID %d with error %d\n",_wtoi(argv[1]),x);

            return 1;
        }
        LPVOID LoadLibraryAddr = (LPVOID)GetProcAddress(GetModuleHandleA("kernel32.dll"),
            "LoadLibraryA");
        if(LoadLibraryAddr == NULL)
        {
            printf("GET PROC ADDRESS FAILED on PID %s\n",argv[1]);
            return 1;
        }
        LPVOID LLParam = (LPVOID)VirtualAllocEx(hProcess, NULL, strlen(FullPath),
            MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
        if(LLParam == NULL)
        {
            printf("VirtualAllocEx on PID %s\n",argv[1]);
            return 1;
        }
        WriteProcessMemory(hProcess, LLParam, FullPath, strlen(FullPath), NULL);
        CreateRemoteThread(hProcess, NULL, NULL, (LPTHREAD_START_ROUTINE)LoadLibraryAddr,
            LLParam, NULL, NULL);
        CloseHandle(hProcess);
        delete [] DirPath;
        delete [] FullPath;
}

BOOL SetPrivilege(
    HANDLE hToken,          // token handle
    LPCTSTR Privilege,      // Privilege to enable/disable
    BOOL bEnablePrivilege   // TRUE to enable.  FALSE to disable
    )
{
    TOKEN_PRIVILEGES tp;
    LUID luid;
    TOKEN_PRIVILEGES tpPrevious;
    DWORD cbPrevious=sizeof(TOKEN_PRIVILEGES);

    if(!LookupPrivilegeValue( NULL, Privilege, &luid )) return FALSE;

    // 
    // first pass.  get current privilege setting
    // 
    tp.PrivilegeCount           = 1;
    tp.Privileges[0].Luid       = luid;
    tp.Privileges[0].Attributes = 0;

    AdjustTokenPrivileges(
            hToken,
            FALSE,
            &tp,
            sizeof(TOKEN_PRIVILEGES),
            &tpPrevious,
            &cbPrevious
            );

    if (GetLastError() != ERROR_SUCCESS) return FALSE;

    // 
    // second pass.  set privilege based on previous setting
    // 
    tpPrevious.PrivilegeCount       = 1;
    tpPrevious.Privileges[0].Luid   = luid;

    if(bEnablePrivilege) {
        tpPrevious.Privileges[0].Attributes |= (SE_PRIVILEGE_ENABLED);
    }
    else {
        tpPrevious.Privileges[0].Attributes ^= (SE_PRIVILEGE_ENABLED &
            tpPrevious.Privileges[0].Attributes);
    }

    AdjustTokenPrivileges(
            hToken,
            FALSE,
            &tpPrevious,
            cbPrevious,
            NULL,
            NULL
            );

    if (GetLastError() != ERROR_SUCCESS) return FALSE;

    return TRUE;
}
/*
BOOL SetPrivilege( 
    HANDLE hToken,  // token handle 
    LPCTSTR Privilege,  // Privilege to enable/disable 
    BOOL bEnablePrivilege  // TRUE to enable. FALSE to disable 
) 
{ 
    TOKEN_PRIVILEGES tp = { 0 }; 
    // Initialize everything to zero 
    LUID luid; 
    DWORD cb=sizeof(TOKEN_PRIVILEGES); 
    if(!LookupPrivilegeValue( NULL, Privilege, &luid ))
        return FALSE; 
    tp.PrivilegeCount = 1; 
    tp.Privileges[0].Luid = luid; 
    if(bEnablePrivilege) { 
        tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; 
    } else { 
        tp.Privileges[0].Attributes = 0; 
    } 
    AdjustTokenPrivileges( hToken, FALSE, &tp, cb, NULL, NULL ); 
    if (GetLastError() != ERROR_SUCCESS) 
        return FALSE; 

    return TRUE;
}
*/

Answer 1

You are replacing CreateFile with your myCreate. When fopen calls CreateFile to open the file, it will instead call your myCreate again, which will call fopen, which will call CreateFile and so on until you run out of stack. You could call oldCreateFile to open the file for outputting, but you won't be able to use fprintf etc with it.