In my project, there is a library that has code to load an fbx using the FBX SDK 2017.1 from Autodesk.
Loading the fbx crashes in debug & release. The crash occurs in 2 different ways and what seems to be at random:
The code does contain realloc() calls, specifically in the allocation of buffers used in a custom implementation of an FbxStream
Most of the code path is entirely identical for windows, only a number of platform specific sections have been re-implemented. On windows, it runs as expected.
What strikes me is that if I run the program in either gdb or valgrind, the crash disappears! So I set out to find uninitialized members/values, but so far I could not find anything suspicious. I used CppDepend/CppCheck and VS2012 code analysis, but both came up empty on un-initialized variables/members
To give some background on FBX loading; the FBX SDK has a number of ways to deal with different types of resources (obj, 3ds, fbx,..). They can be loaded from file or from stream. To support large files, the stream option is the more relevant option. The code below is far from perfect, but what interests me mostly at present is the reason why valgrind/gdb would not crash. I've left the SDK documentation on top of ReadString, since it's the most complex one.
class MyFbxStream : public FbxStream{
uint32 m_FormatID;
uint32 m_Error;
EState m_State;
size_t m_Pos;
size_t m_Size;
const Engine::Buffer* const m_Buffer;
MyFbxStream& operator = (const MyFbxStream& other) const;
public:
MyFbxStream(const Engine::Buffer* const buffer)
: m_FormatID(0)
, m_Error(0)
, m_State(eClosed)
, m_Pos(0)
, m_Size(0)
, m_Buffer(buffer) {};
virtual ~MyFbxStream() {};
virtual bool Open(void* pStreamData) {
m_FormatID = *(uint32*)pStreamData;
m_Pos = 0;
m_State = eOpen;
m_Size = m_Buffer->GetSize();
return true;
}
virtual bool Close() {
m_Pos = m_Size = 0;
m_State = eClosed;
return true;
}
virtual int Read(void* pData, int pSize) const {
const unsigned char* data = (m_Buffer->GetBase(m_Pos));
const size_t bytesRead = m_Pos + pSize > m_Buffer->GetSize() ? (m_Buffer->GetSize() - m_Pos) : pSize;
const_cast<MyFbxStream*>(this)->m_Pos += bytesRead;
memcpy(pData, data, bytesRead);
return (int)bytesRead;
}
/** Read a string from the stream.
* The default implementation is written in terms of Read() but does not cope with DOS line endings.
* Subclasses may need to override this if DOS line endings are to be supported.
* \param pBuffer Pointer to the memory block where the read bytes are stored.
* \param pMaxSize Maximum number of bytes to be read from the stream.
* \param pStopAtFirstWhiteSpace Stop reading when any whitespace is encountered. Otherwise read to end of line (like fgets()).
* \return pBuffer, if successful, else NULL.
* \remark The default implementation terminates the \e pBuffer with a null character and assumes there is enough room for it.
* For example, a call with \e pMaxSize = 1 will fill \e pBuffer with the null character only. */
virtual char* ReadString(char* pBuffer, int pMaxSize, bool pStopAtFirstWhiteSpace = false) {
assert(!pStopAtFirstWhiteSpace); // "Not supported"
const size_t pSize = pMaxSize - 1;
if (pSize) {
const char* const base = (const char* const)m_Buffer->GetBase();
char* cBuffer = pBuffer;
const size_t totalSize = std::min(m_Buffer->GetSize(), (m_Pos + pSize));
const char* const maxSize = base + totalSize;
const char* sum = base + m_Pos;
bool done = false;
// first align the copy on alignment boundary (4byte)
while ((((size_t)sum & 0x3) != 0) && (sum < maxSize)) {
const unsigned char c = *sum++;
*cBuffer++ = c;
if ((c == '\n') || (c == '\r')) {
done = true;
break;
} }
// copy from alignment boundary to boundary (4byte)
if (!done) {
int64 newBytesRead = 0;
uint32* dBuffer = (uint32*)cBuffer;
const uint32* dBase = (uint32*)sum;
const uint32* const dmaxSize = ((uint32*)maxSize) - 1;
while (dBase < dmaxSize) {
const uint32 data = *(const uint32*const)dBase++;
*dBuffer++ = data;
if (((data & 0xff) == 0x0a) || ((data & 0xff) == 0x0d)) { // third bytes, 4 bytes read..
newBytesRead -= 3;
done = true;
break;
} else {
const uint32 shiftedData8 = data & 0xff00;
if ((shiftedData8 == 0x0a00) || (shiftedData8 == 0x0d00)) { // third bytes, 3 bytes read..
newBytesRead -= 2;
done = true;
break;
} else {
const uint32 shiftedData16 = data & 0xff0000;
if ((shiftedData16 == 0x0a0000) || (shiftedData16 == 0x0d0000)) { // second byte, 2 bytes read..
newBytesRead -= 1;
done = true;
break;
} else {
const uint32 shiftedData24 = data & 0xff000000;
if ((shiftedData24 == 0x0a000000) || (shiftedData24 == 0x0d000000)) { // first byte, 1 bytes read..
done = true;
break;
} } } } }
newBytesRead += (int64)dBuffer - (int64)cBuffer;
if (newBytesRead) {
sum += newBytesRead;
cBuffer += newBytesRead;
} }
// copy anything beyond the last alignment boundary (4byte)
if (!done) {
while (sum < maxSize) {
const unsigned char c = *sum++;
*cBuffer++ = c;
if ((c == '\n') || (c == '\r')) {
done = true;
break;
} } }
const size_t bytesRead = cBuffer - pBuffer;
if (bytesRead) {
const_cast<MyFbxStream*>(this)->m_Pos += bytesRead;
pBuffer[bytesRead] = 0;
return pBuffer;
} }
pBuffer = NULL;
return NULL;
}
virtual void Seek(const FbxInt64& pOffset, const FbxFile::ESeekPos& pSeekPos) {
switch (pSeekPos) {
case FbxFile::ESeekPos::eBegin: m_Pos = pOffset; break;
case FbxFile::ESeekPos::eCurrent: m_Pos += pOffset; break;
case FbxFile::ESeekPos::eEnd: m_Pos = m_Size - pOffset; break;
}
}
virtual long GetPosition() const { return (long)m_Pos; }
virtual void SetPosition(long position) { m_Pos = position; }
virtual void ClearError() { m_Error = 0; }
virtual int GetError() const { return m_Error; }
virtual EState GetState() { return m_State; }
virtual int GetReaderID() const { return m_FormatID; }
virtual int GetWriterID() const { return -1; } // readonly stream
virtual bool Flush() { return true; } // readonly stream
virtual int Write(const void* /*d*/, int /*s*/) { assert(false); return 0; } // readonly stream
};
I assume that there may be undefined behavior related to malloc/free/realloc operations that somehow do not occur in gdb. But if this is the case, I also expect the Windows binaries to have problems.
Also, I don't know if this is relevant, but the when I trace into the Open() function and print the "m_Buffer" pointer's value (or the "this"), I get a pointer value starting with 0xfffffff.. which for a Windows programmer looks like a problem. However, can I pull the same conclusion in linux, since I also saw this happening in static function calls etc.
if I run the program in either gdb or valgrind, the crash disappears!
There are two possible explanations:
The steps I would take: