C++ WinHTTP InternetReadFile not retrieving entire source

Question

I am trying to get the content of a webpage using WinHTTP in C++. It is kinda working except i am on able to view parts of the source not the entire source.

The OpenSSL stuff is there because eventually i want to be able to do HTTP over SSL, but that's the next challenge.

#include <windows.h>
#include <Wininet.h>

#pragma comment(lib, "Crypt32.lib")
#pragma comment(lib, "wininet.lib")


bool SetupSSL(HINTERNET request)
{
    HCERTSTORE store = CertOpenSystemStore(NULL,"CA");
    DWORD ret = 0;
    bool ok = false;

    if(store==NULL)     
        return false;
    PCCERT_CONTEXT context = CertFindCertificateInStore(store,X509_ASN_ENCODING,0,CERT_FIND_SUBJECT_STR,L"WebClient",NULL);
    if(context!=NULL)
    {            
        ok = InternetSetOption(request,INTERNET_OPTION_CLIENT_CERT_CONTEXT,(LPVOID)context,sizeof(CERT_CONTEXT))==TRUE;
        if(!ok)         
            ret = GetLastError();  
        CertFreeCertificateContext(context);
    }
    MessageBoxA(0, "1", 0, 0);
    CertCloseStore(store,0);
    return ok;
};


int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPTSTR lpCmdLine, int nCmdShow)
{   
    HINTERNET hint = InternetOpen("WebTestClient",INTERNET_OPEN_TYPE_DIRECT,NULL,NULL,0);
    if(hint!=NULL)
    {
        //InternetSetStatusCallback(hint,&HTTPStatusCallbackFunc);
        HINTERNET hsession = InternetConnect(hint,"encrypted.google.com",443,NULL,NULL,3,0,NULL);

        if(hsession!=NULL)
        {
            HINTERNET hreqest = HttpOpenRequest(hsession,"GET","/","HTTP/1.1",NULL,NULL,INTERNET_FLAG_SECURE|INTERNET_FLAG_NO_AUTH,NULL);
            if(hreqest!=NULL)
            {
                //if(SetupSSL(hreqest))
                //{ 

                    if(HttpSendRequest(hreqest,NULL,0, NULL, 0))
                    {                   
                        //char szResponse[1024] = "";
                        char szBuffer[1024] = "";
                        DWORD dwRead=0;                 


                        while(InternetReadFile(hreqest, szBuffer, sizeof(szBuffer)-1, &dwRead) && dwRead) 
                        {
                            //strcat(szBuffer, szResponse);
                            szBuffer[dwRead] = 0;
                            OutputDebugStringA(szBuffer);               
                            dwRead=0;                       
                        }                                       

                        //InternetReadFile(hreqest, szBuffer, sizeof(szBuffer)-1, &dwRead);
                        MessageBoxA(0, szBuffer, 0, 0); // Only shows the first part of the source.
                    }               

                //}
                //else
                //{
                    //MessageBoxA(0, "SetupSSL failed", 0, 0);
                //}
                InternetCloseHandle(hreqest);
            }       
            InternetCloseHandle(hsession);
        }       
        InternetCloseHandle(hint);
    }

    return 0;
}

OutPutDebugStringA is outputting the entire source to the debug console, so it's ready it all. Just not storing it all in the szBuffer variable. I think it might be something to do with the loop.

Answer 1

while(InternetReadFile(hreqest, szBuffer, sizeof(szBuffer)-1, ...

You keep reading using InternetReadFile into the same buffer szBuffer overwriting stuff you already having there.

Typically you would want a flexible/reallocated buffer here, or a list of buffers, and just for code brevity here, here is what you can do to sequentially read into large enough fixed size buffer:

static const SIZE_T g_nBufferCapacity = 128 * 1024; // 128 KB
CHAR szBuffer[nBufferCapacity] = "";
SIZE_T nBufferSize = 0;

szBuffer[0] = 0;
for(; ; )
{
  DWORD nReadableSize = min(1024, sizeof(szBuffer) - nBufferSize - 1);
  DWORD nReadSize = 0;
  InternetReadFile(hreqest, szBuffer + nBufferSize, nReadableSize, &nReadSize); // TODO: Error Checking
  if(!nReadSize)
    break;
  szBuffer[nBufferSize + nReadSize] = 0;
  OutputDebugStringA(szBuffer + nBufferSize);               
  nBufferSize += nReadSize;
}