Use of GL_STENCIL_TEST to render concave polygons

Question

I'm working on a custom geometry library adapted to Quartz Composer and trying to draw some concave polygons in a Plug In.

I implemented the poly2tri library, so the user can choose to triangulate or not, but it's not suitable for a per-frame polygon transformations rendering.

I'm a noob in OpenGL and I've been reading and testing a lot, about stencil buffer and odd/even operations, but even code that seem to work for other people, doesn't work for me.

The render context is a CGLContextObj, and I'm working on a MacBook Pro Retina Display, with NVidia GEForce GT650. I read that all configurations don't have stencil buffers, but it look like it works sometimes, though not as I would like it to.

I was wondering if someone with the same kind of config was using a code that works and could take a look at my code. In particular, I'm curious too about the number of passes requested, according to the number of vertices or "convexity defects" I guess...

I took my infos from :

• http://fly.cc.fer.hr/~unreal/theredbook/chapter13.html
• http://commaexcess.com/articles/7/concave-polygon-triangulation-shortcut
• http://graphicsbb.itgo.com/solutions/extrude.html
• http://analysesmusings.wordpress.com/2012/07/13/drawing-filled-concave-polygons-using-the-stencil-buffer/

... but still not clear...

Here is my code (one of them in fact, as I tested so much configurations) and a picture of the result. Actually I use to put the actual rendering in a method called for each polygon, but I rewrote it to be much clear :

---

EDIT

In fact I understood that I have to draw each triangle, in order to invert the bit value in the stencil buffer. So I rewrote my code into this :

CGLContextObj cgl_ctx = [context CGLContextObj];

CGLLockContext(cgl_ctx);

GLenum                          error;

if(cgl_ctx == NULL)
    return NO;


glPushAttrib(GL_ALL_ATTRIB_BITS);
glPushClientAttrib(GL_CLIENT_ALL_ATTRIB_BITS);



glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);

glDisable(GL_CULL_FACE);
glClear(GL_STENCIL_BUFFER_BIT);
glClearStencil(0);

glEnable(GL_STENCIL_TEST);

glStencilOp(GL_INVERT, GL_INVERT, GL_INVERT);
glStencilFunc(GL_ALWAYS, 1, 1);
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);

// glColor4d(1., 1., 1., 1.);        ----> does it make sense ?
glBegin(GL_TRIANGLE_FAN); {

    for (int i = 1; i < [vertices count] - 1; i++) {
        // Allways drawing the first vertex
        glVertex2d([[[vertices objectAtIndex:0] objectAtIndex:0] doubleValue], [[[vertices objectAtIndex:0] objectAtIndex:1] doubleValue]);

        // Then two others to make a triangle
        glVertex2d([[[vertices objectAtIndex:i] objectAtIndex:0] doubleValue], [[[vertices objectAtIndex:i] objectAtIndex:1] doubleValue]);
        glVertex2d([[[vertices objectAtIndex:i+1] objectAtIndex:0] doubleValue], [[[vertices objectAtIndex:i+1] objectAtIndex:1] doubleValue]);

    }

}
glEnd();


glStencilOp(GL_ZERO, GL_ZERO, GL_ZERO);
glStencilFunc(GL_EQUAL, 1, 1);
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);

glColor4d(1., 0., 0., 0.5);
glBegin(GL_TRIANGLE_FAN); {
    for (id vertex in vertices) {
        glVertex2d([[vertex objectAtIndex:0] doubleValue], [[vertex objectAtIndex:1] doubleValue]);
    }
    glVertex2d([[[vertices objectAtIndex:0] objectAtIndex:0] doubleValue], [[[vertices objectAtIndex:0] objectAtIndex:1] doubleValue]);
}
glEnd();


glDisable (GL_STENCIL_TEST);




glDisable(GL_BLEND);


glPopClientAttrib();
glPopAttrib();



if((error = glGetError()))
    NSLog(@"OpenGL error %04X", error);

CGLUnlockContext(cgl_ctx);

return (error ? NO : YES);

But it still doesn't work. Here is my result and the link to the original image and the explanation.

http://what-when-how.com/opengl-programming-guide/drawing-filled-concave-polygons-using-the-stencil-buffer-opengl-programming/

EDIT 2 :

In fact, the context enabled by Quartz Composer doesn't implement a stencil buffer. It seems impossible to render directly in OpenGL with the stencil buffer.

Answer 1

...
glClearStencil(0);
...

Be aware that glClearStencil() just sets a bit of state and doesn't actually clear the stencil buffer.

Try adding a glClear( GL_STENCIL_BUFFER_BIT ) somewhere before each polygon.

EDIT: Like this:

#include <GL/glut.h>
#include <glm/glm.hpp>
#include <glm/gtc/type_ptr.hpp>
#include <vector>

std::vector< glm::vec2 > pts;
bool leftHeld = true;
glm::vec2* dragPt = NULL;
void mouse( int button, int state, int x, int y )
{
    glm::vec2 pt( x, glutGet( GLUT_WINDOW_HEIGHT ) - y );

    // left mouse button starts dragging a point
    dragPt = NULL;
    leftHeld = false;
    if( button == GLUT_LEFT_BUTTON && state == GLUT_DOWN )
    {
        leftHeld = true;
        size_t minIdx = 0;
        for( size_t i = 0; i < pts.size(); ++i )
        {
            float newDist = glm::distance( pt, pts[ i ] );
            float oldDist = glm::distance( pt, pts[ minIdx ] );
            if( newDist <= oldDist && newDist < 15.0f )
            {
                minIdx = i;
                dragPt = &pts[ minIdx ];
            }
        }
    }

    // middle mouse button clears all points
    if( button == GLUT_MIDDLE_BUTTON && state == GLUT_UP )
    {
        pts.clear();
    }

    // right mouse button adds a point
    if( button == GLUT_RIGHT_BUTTON && state == GLUT_UP )
    {
        pts.push_back( pt );
    }

    glutPostRedisplay();
}

void motion( int x, int y )
{
    glm::vec2 pt( x, glutGet( GLUT_WINDOW_HEIGHT ) - y );
    if( dragPt && leftHeld )
    {
        *dragPt = pt;
        glutPostRedisplay();
    }
}

void glLine( const std::vector< glm::vec2 >& line, GLenum mode )
{
    glBegin( mode );
    for( size_t i = 0; i < line.size(); ++i )
    {
        glVertex2f( line[i].x, line[i].y );
    }
    glEnd();
}

void display()
{
    glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT );

    glMatrixMode( GL_PROJECTION );
    glLoadIdentity();
    double w = glutGet( GLUT_WINDOW_WIDTH );
    double h = glutGet( GLUT_WINDOW_HEIGHT );
    glOrtho( 0, w, 0, h, -1, 1 );

    glMatrixMode( GL_MODELVIEW );
    glLoadIdentity();

    // draw polygon
    glClear( GL_STENCIL_BUFFER_BIT );
    {
        // fill stencil buffer
        glEnable( GL_STENCIL_TEST );
        glColorMask( GL_FALSE,GL_FALSE,GL_FALSE,GL_FALSE );
        glStencilOp( GL_KEEP, GL_KEEP, GL_INVERT );
        glStencilFunc( GL_ALWAYS, 0x1, 0x1 );
        glBegin( GL_TRIANGLES );
        for( size_t i = 1; i+1 < pts.size(); ++i )
        {
            glVertex2fv( glm::value_ptr( pts[ 0 ] ) );
            glVertex2fv( glm::value_ptr( pts[ i ] ) );
            glVertex2fv( glm::value_ptr( pts[ i+1 ] ) );
        }
        glEnd();

        // fill color buffer
        glColor3ub( 0, 128, 0 );
        glColorMask( GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE );
        glStencilFunc( GL_EQUAL, 0x1, 0x1 );
        glBegin( GL_TRIANGLES );
        for( size_t i = 1; i+1 < pts.size(); ++i )
        {
            glVertex2fv( glm::value_ptr( pts[ 0 ] ) );
            glVertex2fv( glm::value_ptr( pts[ i ] ) );
            glVertex2fv( glm::value_ptr( pts[ i+1 ] ) );
        }
        glEnd();
        glDisable( GL_STENCIL_TEST );
    }

    // draw polygon boundary
    glLineWidth( 1 );
    glColor3ub( 255, 255, 255 );
    glLine( pts, GL_LINE_LOOP );

    // draw vertexes
    glPointSize( 9 );
    glColor3ub( 255, 0, 0 );
    glLine( pts, GL_POINTS );

    glutSwapBuffers();
}

int main( int argc, char **argv )
{
    glutInit( &argc, argv );
    glutInitDisplayMode( GLUT_RGBA | GLUT_DEPTH | GLUT_DOUBLE | GLUT_STENCIL );
    glutInitWindowSize( 640, 480 );
    glutCreateWindow( "GLUT" );
    glutMouseFunc( mouse );
    glutMotionFunc( motion );
    glutDisplayFunc( display );
    glutMainLoop();
    return 0;
}