From f012e4516c9cbe7b49f1e7b7ec390e0f09d519c1 Mon Sep 17 00:00:00 2001
From: Robert Osfield <robert@openscenegraph.com>
Date: Tue, 29 May 2007 10:31:03 +0000
Subject: [PATCH] From Sherman Wilcox, "The issue itself occurs when a model is
 tessellated that has greater than unsigned short number of vertexes. An
 object called vertexPtrToIndexMap contains vertexes and their indexes. This
 std::map object can obviously hold a quantity that is greater than unsigned
 short, however osg::DrawElementsUShort objects were being created to
 reference these vertexes and their indexes. osg::DrawElementsUShort can only
 hold indexes that are 16-bit quantities.

...

proposed_patch_2\tessellator.cpp. This solution examines the size of
vertexPtrToIndexMap and selects 1 of 3 possible osg::DrawElements
objects: DrawElementsUByte, DrawElementsUShort, or DrawElementsUInt.
The main drawback of this particular solution is the code duplication.
However, the repair is straightforward."
---
 src/osgUtil/Tessellator.cpp | 55 +++++++++++++++++++++++++++++--------
 1 file changed, 44 insertions(+), 11 deletions(-)

diff --git a/src/osgUtil/Tessellator.cpp b/src/osgUtil/Tessellator.cpp
index 7e658ec4f..7218e85e8 100644
--- a/src/osgUtil/Tessellator.cpp
+++ b/src/osgUtil/Tessellator.cpp
@@ -702,18 +702,52 @@ void Tessellator::collectTessellation(osg::Geometry &geom, unsigned int original
         primItr!=_primList.end();
         ++primItr, ++_index)
         {
-              Prim* prim = primItr->get();
-              
-              osg::DrawElementsUShort* elements = new osg::DrawElementsUShort(prim->_mode);
-              for(Prim::VecList::iterator vitr=prim->_vertices.begin();
-              vitr!=prim->_vertices.end();
-              ++vitr)
+              Prim* prim=primItr->get();
+              int ntris=0;
+
+              if(vertexPtrToIndexMap.size() <= 255)
               {
-                  elements->push_back(vertexPtrToIndexMap[*vitr]);
+                  osg::DrawElementsUByte* elements = new osg::DrawElementsUByte(prim->_mode);
+                  for(Prim::VecList::iterator vitr=prim->_vertices.begin();
+                  vitr!=prim->_vertices.end();
+                  ++vitr)
+                {
+                    elements->push_back(vertexPtrToIndexMap[*vitr]);
+                }
+
+                  // add to the drawn primitive list.
+                  geom.addPrimitiveSet(elements);
+                  ntris=elements->getNumIndices()/3;
               }
-              
-              // add to the drawn primitive list.
-              geom.addPrimitiveSet(elements);
+              else if(vertexPtrToIndexMap.size() > 255 && vertexPtrToIndexMap.size() <= 65535)
+              {
+                  osg::DrawElementsUShort* elements = new osg::DrawElementsUShort(prim->_mode);
+                  for(Prim::VecList::iterator vitr=prim->_vertices.begin();
+                    vitr!=prim->_vertices.end();
+                    ++vitr)
+                  {
+                    elements->push_back(vertexPtrToIndexMap[*vitr]);
+                  }
+
+                  // add to the drawn primitive list.
+                  geom.addPrimitiveSet(elements);
+                  ntris=elements->getNumIndices()/3;
+              }
+              else
+              {
+                  osg::DrawElementsUInt* elements = new osg::DrawElementsUInt(prim->_mode);
+                  for(Prim::VecList::iterator vitr=prim->_vertices.begin();
+                    vitr!=prim->_vertices.end();
+                    ++vitr)
+                  {
+                    elements->push_back(vertexPtrToIndexMap[*vitr]);
+                  }
+
+                  // add to the drawn primitive list.
+                  geom.addPrimitiveSet(elements);
+                  ntris=elements->getNumIndices()/3;
+              }
+
               if (primItr==_primList.begin()) 
               {   // first primitive so collect primitive normal & colour.
                   if (normals) {
@@ -754,7 +788,6 @@ void Tessellator::collectTessellation(osg::Geometry &geom, unsigned int original
                     if (cols3) cols3->push_back(primCol3); // GWM Dec 2003 add flat shaded colour for new facet
                   }
                   if (prim->_mode==GL_TRIANGLES) {
-                      int ntris=elements->getNumIndices()/3;
                       if (geom.getNormalBinding()==osg::Geometry::BIND_PER_PRIMITIVE_SET ||
                           geom.getNormalBinding()==osg::Geometry::BIND_PER_PRIMITIVE) { // need one per triangle? Not one per set.
                           for (int ii=1; ii<ntris; ii++) {