glutesselator always crashes at gluTessEndPolygon()

Question

I use glu tessellation to tessellate complex polygons. The code simplified is listed bellow.

It always crashes at gluTessEndPolygon(GLUtessobj) with error:

Error: 0xC0000005: Access violation reading location 0x57783b39;

The code works when the number of points of the polygon is small (<100).

I just can't figure out why.

typedef boost::geometry::model::point<float, 2, boost::geometry::cs::cartesian> pt;
typedef boost::geometry::model::polygon<pt> Polygon;
typedef boost::geometry::model::ring<pt> Ring;
vector<Polygon> g_myPolys;


// ------Static variables used in glu tessellation------
static GLUtesselator  *GLUtessobj;
static unsigned int   s_gltri_type;
static int s_tess_orient;
static int s_cur_pt_idx;
//    Create an array to hold pointers to allocated vertices created by "combine" callback,
//    so that they may be deleted after tessellation.
static std::vector<GLdouble*>     s_combineVertexArray;

// Store tessellated results
static std::vector<double> s_vecTriVerts;  // Store area objects' tessellated triangle( triangle fan, triangle strip and triangles) vertices.
static std::vector<int> s_vecTriStripVertCnts;   // Store every triangle strips' start indices in m_vecTriVerts.
static std::vector<int> s_vecTriStripFirstIdx;   // Store every triangle strips' vertex count start from its start index.

static std::vector<int> s_vecTriFanVertCnts;  // Store every triangle fans' start indices in m_vecTriVerts.
static std::vector<int> s_vecTriFanFirstIdx;  // Store every triangle fans' vertex count start from its start index.

static std::vector<int> s_vecTrisVertCnts;   // Store every triangles' start indices in m_vecTriVerts.
static std::vector<int> s_vecTrisFirstIdx;   // Store every triangles' vertex count start from its start index.

static int s_cur_tri_fans_vert_cnt;
static int s_cur_tri_strips_vert_cnt;
static int s_cur_tris_vert_cnt;

static std::vector<double*> s_vecTmp;

void beginCallback(GLenum which)
{
    s_gltri_type = which;
    switch ( s_gltri_type)
    {
    case GL_TRIANGLE_FAN:
        s_vecTriFanFirstIdx.push_back(s_cur_pt_idx);
        s_cur_tri_fans_vert_cnt = 0;
        break;
    case GL_TRIANGLE_STRIP:
        s_vecTriStripFirstIdx.push_back(s_cur_pt_idx);
        s_cur_tri_strips_vert_cnt = 0;
        break;
    case GL_TRIANGLES:
        s_vecTrisFirstIdx.push_back(s_cur_pt_idx);
        s_cur_tris_vert_cnt = 0;
        break;
    }
}


void vertexCallback(GLvoid *vertex)
{
    GLdouble *pv = (GLdouble *) vertex;
    s_vecTriVerts.push_back(pv[0]);
    s_vecTriVerts.push_back(pv[1]);

    s_cur_pt_idx ++;

    switch ( s_gltri_type)
    {
    case GL_TRIANGLE_FAN:
        s_cur_tri_fans_vert_cnt ++;
        break;
    case GL_TRIANGLE_STRIP:
        s_cur_tri_strips_vert_cnt ++;
        break;
    case GL_TRIANGLES:
        s_cur_tris_vert_cnt ++;
        break;
    }
}

void combineCallback(GLdouble coords[3],
    GLdouble *vertex_data[4],
    GLfloat weight[4], GLdouble **dataOut )
{
    GLdouble *vertex = (GLdouble *)malloc(6 * sizeof(GLdouble));

    vertex[0] = coords[0];
    vertex[1] = coords[1];
    vertex[2] = coords[2];
    vertex[3] = vertex[4] = vertex[5] = 0.0;

    *dataOut = vertex;

    s_combineVertexArray.push_back(vertex);
}

void endCallback()
{
    switch ( s_gltri_type)
    {
    case GL_TRIANGLE_FAN:
        s_vecTriFanVertCnts.push_back(s_cur_tri_fans_vert_cnt);
        break;
    case GL_TRIANGLE_STRIP:
        s_vecTriStripVertCnts.push_back(s_cur_tri_strips_vert_cnt);
        break;
    case GL_TRIANGLES:
        s_vecTrisVertCnts.push_back(s_cur_tris_vert_cnt);
        break;
    }
}

void errorCallback(GLenum errorCode)
{
    const GLubyte *estring;
    estring = gluErrorString(errorCode);
    printf ("Tessellation Error: %s\n", estring);
}

void Tessellate()
{
    // Create tessellate object
    GLUtessobj = gluNewTess();

    //  Register the callbacks
    gluTessCallback(GLUtessobj, GLU_TESS_BEGIN,   (void (__stdcall*)())&beginCallback);
    gluTessCallback(GLUtessobj, GLU_TESS_VERTEX,  (void (__stdcall*)())&vertexCallback);
    gluTessCallback(GLUtessobj, GLU_TESS_END,     (void (__stdcall*)())&endCallback);
    gluTessCallback(GLUtessobj, GLU_TESS_COMBINE, (void (__stdcall*)())&combineCallback);
    gluTessCallback(GLUtessobj, GLU_TESS_ERROR,   (void (__stdcall*)())&errorCallback);

    gluTessProperty(GLUtessobj, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_POSITIVE );

    gluTessBeginPolygon(GLUtessobj, NULL);
    gluTessBeginContour(GLUtessobj);

    Polygon pp =  g_myPolys[0];
    for ( int i = 0; i < pp.outer().size(); i ++)
    {
        GLdouble  *p = new GLdouble[3];
        s_vecTmp.push_back(p);

        p[0] = pp.outer()[i].get<0>();
        p[1] = pp.outer()[i].get<1>();
        p[2] = 0.0;

        gluTessVertex( GLUtessobj, p, p ) ;
    }

    gluTessEndContour(GLUtessobj);
    gluTessEndPolygon(GLUtessobj);

    gluDeleteTess(GLUtessobj);

    for ( int i = 0; i < s_vecTmp.size(); i ++)
        delete[] s_vecTmp[i];
    s_vecTmp.clear();

    // Free up any "Combine" vertices created
    for(unsigned int i = 0; i < s_combineVertexArray.size(); i++)
        free (s_combineVertexArray[i]);
    s_combineVertexArray.clear();
}

Answer 1

One thing that immediately strikes me as odd is, that you do the cast to __stdcall there.

gluTessCallback(GLUtessobj, GLU_TESS_BEGIN,   (void (__stdcall*)())&beginCallback);

Why are you doing that? If your compiler complains about incompatible calling conventions, then the last thing you should do there is casting the calling convention. Only despair and horror await if you cast a calling convention. It's already a bad idea to cast pointers (in C++ casting from/to void* is kind of okay, but that's it).

And then there are a few other weird things you do with pointers. For example you're mixing std::vector with manually managed memory (new GLdouble[3]). Seriously, why?!

I strongly suggest you simplify your data structures and clean up that pointer juggling. Most likely you have some out of bounds buffer write somewhere in your code, but it's difficult to see where exactly.