Strange behaviour of gluUnProject trying to convert mouse coords to opengl coords

Question

I tried to convert the mouse coords in click to opengl coords. Actually it seems to work, but when I uncomment a cout-line that I just wrote for testing it sets my vars to not a number -nan. How does it happen? How can I fix it?

//global:
GLdouble mouseOgl[3] = {0.0,0.0,0.0};
//handle click events of the mouse
void myMouse(int button, int state, int x, int y)
{
    //mouse coords to gl coords


    switch (button)
    {
        case GLUT_LEFT_BUTTON:
            if(state == GLUT_UP){   //on release left mouse button
                std::cout << x << " * "<< y << std::endl;
                GetOGLPos(x, y);
                std::cout
                    << mouseOgl[0] << " # "
                    << mouseOgl[1] << " # "
                    << mouseOgl[2] << " # "
                    << std::endl;   
                glutPostRedisplay();            
                }
        break;
    }
}

whole code here:

#include <iostream>
#include <stdlib.h>
#include <string>
#include <fstream>
#include <math.h>
#include <time.h> 
#include <GL/glut.h>

const GLint nNumPoints = 5;

GLfloat ctrlpoints[5][3] = {
        { -4.0, -4.0, 0.0}, { -2.0, 4.0, 0.0}, 
        {2.0, -4.0, 0.0}, {4.0, 4.0, 0.0}, {6.0, 2.0, 0.0}};
GLdouble mouseOgl[3] = {0.0,0.0,0.0};

void GetOGLPos(int x, int y);

void init(void)
{
   glClearColor(1.0, 1.0, 1.0, 0.0);
   glShadeModel(GL_FLAT);
   glMap1f(GL_MAP1_VERTEX_3,    // Type of data generated
    0.0f,           // Lower u range
    1.0f,           // Upper u range
    3,              // Distance between points in the data
    nNumPoints,         // number of control points
    &ctrlpoints[0][0]);     // array of control points
    // Enable the evaluator
    glEnable(GL_MAP1_VERTEX_3);
    glEnable(GL_DEPTH);
}

void display(void)
{
   int i;
   glClear(GL_COLOR_BUFFER_BIT);
   glColor3f(0.0, 0.0, 1.0);
   glBegin(GL_LINE_STRIP);
      for (i = 0; i <= 30; i++) 
         glEvalCoord1f((GLfloat) i/30.0);
   glEnd();
   //Kontrollpunkte:
   glPointSize(5.0);
   glColor3f(1.0, 0.0, 0.0);
   glBegin(GL_POINTS);
      for (i = 0; i < nNumPoints; i++) 
         glVertex3fv(&ctrlpoints[i][0]);
   glEnd();
   glColor3f(0.0, 1.0, 0.0);
   glBegin(GL_LINE_STRIP);
        for (i = 0; i < nNumPoints; i++) 
            glVertex3fv(&ctrlpoints[i][0]);
   glEnd();
   glFlush();
}

void reshape(int w, int h)
{
   glViewport(0, 0, (GLsizei) w, (GLsizei) h);
   glMatrixMode(GL_PROJECTION);
   glLoadIdentity();
   //keep aspect ratio:
   if (w <= h)
      glOrtho(-10.0, 10.0, -10.0*(GLfloat)h/(GLfloat)w, 
               10.0*(GLfloat)h/(GLfloat)w, -10.0, 10.0);
   else
      glOrtho(-10.0*(GLfloat)w/(GLfloat)h, 
               10.0*(GLfloat)w/(GLfloat)h, -10.0, 10.0, -10.0, 10.0);
   glMatrixMode(GL_MODELVIEW);
   glLoadIdentity();
}

//handle click events of the mouse
void myMouse(int button, int state, int x, int y)
{
    //mouse coords to gl coords


    switch (button)
    {
        case GLUT_LEFT_BUTTON:
            if(state == GLUT_UP){   //on release left mouse button
                std::cout << x << " * "<< y << std::endl;
                GetOGLPos(x, y);
                std::cout
                    << mouseOgl[0] << " # "
                    << mouseOgl[1] << " # "
                    << mouseOgl[2] << " # "
                    << std::endl;   
                glutPostRedisplay();            
                }
        break;
    }
}
// detailed information: 
// http://nehe.gamedev.net/article/using_gluunproject/16013/
void GetOGLPos(int x, int y)
{
    //init vars:
    GLint viewport[4];          
    GLdouble modelview[16];     
    GLdouble projection[16];    
    GLfloat winX, winY, winZ;   
    GLdouble posX, posY, posZ;
    //get gl specs
    glGetDoublev( GL_MODELVIEW_MATRIX, modelview ); //get Modelmatrix   
    glGetDoublev( GL_PROJECTION_MATRIX, projection );   //get projection matrix
    glGetIntegerv( GL_VIEWPORT, viewport );     //get viewport values
    //calculate the gl mouseposition
    winX = (float)x;
    winY = (float)viewport[3] - (float)y;
    glReadPixels( x, int(winY), 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &winZ );

    gluUnProject( winX, winY, winZ, modelview, projection, viewport, &posX, &posY, &posZ);
/*
    following line needed to run the program propper?!?!
*/
    std::cout << "positions:" << posX << " | " << posY << " | " << posZ << std::endl;
    mouseOgl[0] = posX;
    mouseOgl[1] = posY;
    mouseOgl[2] = posZ;
}

int main(int argc, char** argv)
{
   glutInit(&argc, argv);
   glutInitDisplayMode (GLUT_SINGLE | GLUT_RGB);
   glutInitWindowSize (600, 600);
   glutInitWindowPosition (100, 100);
   glutCreateWindow (argv[0]);
   init ();
   glutDisplayFunc(display);
   glutReshapeFunc(reshape);
   glutMouseFunc(myMouse);
   glutMainLoop();
   return 0;
}

Answer 1

mouse callback can be called any time, so you are not sure what is the proper state of your rendering code when you are in that callback...

I think You should mark that the mouse was pressed (save it to some variable wasMousePressed = true;) in this callback and then check mouse hits in your OnRender function. That way it will be synchronized with the opengl. Then check if your cout code works properly.

onMouse() {
    if (...)
        mousePressed = true;
    else
        mousePressed = false;
}

onRender() {
  clearBuffer();

  setupCameraAndProjection();

   if (mousePressed)
     checkOGLState();

   render...
}